| 17 | |
| 18 | |
| 19 | def fetch_ips(q: Queue, validator_queue: Queue): |
| 20 | logger.debug('fetch_ips...') |
| 21 | worker = Worker() |
| 22 | |
| 23 | while True: |
| 24 | try: |
| 25 | provider: BaseProvider = q.get()() |
| 26 | |
| 27 | provider_name = provider.__class__.__name__ |
| 28 | |
| 29 | logger.debug('Get a provider from the provider queue: ' + provider_name) |
| 30 | |
| 31 | for url in provider.urls(): |
| 32 | try: |
| 33 | html = worker.get_html(url, render_js=provider.should_render_js()) |
| 34 | except Exception as e: |
| 35 | logger.error("worker.get_html failed: %s", e) |
| 36 | continue |
| 37 | |
| 38 | if html: |
| 39 | proxies = provider.parse(html) |
| 40 | |
| 41 | for p in proxies: |
| 42 | validator_queue.put(p) |
| 43 | # logger.debug('Put new proxy ip into queue: {}'.format(p.__str__())) |
| 44 | |
| 45 | logger.info( |
| 46 | ' {}: feed {} potential proxies into the validator queue'.format(provider_name, len(proxies)) |
| 47 | ) |
| 48 | except (KeyboardInterrupt, InterruptedError, SystemExit): |
| 49 | worker.stop() |
| 50 | logger.info('worker_process exited.') |
| 51 | break |
| 52 | |
| 53 | |
| 54 | def validate_ips(validator_queue: Queue, validator_pool: ThreadPoolExecutor): |