| 87 | |
| 88 | |
| 89 | class SpiderThread(Thread): |
| 90 | |
| 91 | def __init__(self, name, spider, tasks_queue): |
| 92 | super().__init__(name=name, daemon=True) |
| 93 | self.spider = spider |
| 94 | self.tasks_queue = tasks_queue |
| 95 | |
| 96 | def run(self): |
| 97 | while True: |
| 98 | current_url = self.tasks_queue.get() |
| 99 | visited_urls.add(current_url) |
| 100 | self.spider.status = SpiderStatus.WORKING |
| 101 | html_page = self.spider.fetch(current_url) |
| 102 | if html_page not in [None, '']: |
| 103 | url_links = self.spider.parse(html_page) |
| 104 | for url_link in url_links: |
| 105 | self.tasks_queue.put(url_link) |
| 106 | self.spider.status = SpiderStatus.IDLE |
| 107 | |
| 108 | |
| 109 | def is_any_alive(spider_threads): |