| 99 | self.spider = spider |
| 100 | |
| 101 | def run(self): |
| 102 | redis_client = redis.Redis(host='1.2.3.4', port=6379, password='1qaz2wsx') |
| 103 | mongo_client = pymongo.MongoClient(host='1.2.3.4', port=27017) |
| 104 | thread_local.redis_client = redis_client |
| 105 | thread_local.mongo_db = mongo_client.msohu |
| 106 | while True: |
| 107 | current_url = redis_client.lpop('m_sohu_task') |
| 108 | while not current_url: |
| 109 | current_url = redis_client.lpop('m_sohu_task') |
| 110 | self.spider.status = SpiderStatus.WORKING |
| 111 | current_url = current_url.decode('utf-8') |
| 112 | if not redis_client.sismember('visited_urls', current_url): |
| 113 | redis_client.sadd('visited_urls', current_url) |
| 114 | html_page = self.spider.fetch(current_url) |
| 115 | if html_page not in [None, '']: |
| 116 | hasher = hasher_proto.copy() |
| 117 | hasher.update(current_url.encode('utf-8')) |
| 118 | doc_id = hasher.hexdigest() |
| 119 | sohu_data_coll = mongo_client.msohu.webpages |
| 120 | if not sohu_data_coll.find_one({'_id': doc_id}): |
| 121 | sohu_data_coll.insert_one({ |
| 122 | '_id': doc_id, |
| 123 | 'url': current_url, |
| 124 | 'page': Binary(zlib.compress(pickle.dumps(html_page))) |
| 125 | }) |
| 126 | self.spider.parse(html_page) |
| 127 | self.spider.status = SpiderStatus.IDLE |
| 128 | |
| 129 | |
| 130 | def is_any_alive(spider_threads): |