getter of proxypool
| 5 | from proxypool.testers import __all__ as testers_cls |
| 6 | |
| 7 | class Getter(object): |
| 8 | """ |
| 9 | getter of proxypool |
| 10 | """ |
| 11 | |
| 12 | def __init__(self): |
| 13 | """ |
| 14 | init db and crawlers |
| 15 | """ |
| 16 | self.redis = RedisClient() |
| 17 | self.crawlers_cls = crawlers_cls |
| 18 | self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls] |
| 19 | self.testers_cls = testers_cls |
| 20 | self.testers = [tester_cls() for tester_cls in self.testers_cls] |
| 21 | |
| 22 | def is_full(self): |
| 23 | """ |
| 24 | if proxypool if full |
| 25 | return: bool |
| 26 | """ |
| 27 | return self.redis.count() >= PROXY_NUMBER_MAX |
| 28 | |
| 29 | @logger.catch |
| 30 | def run(self): |
| 31 | """ |
| 32 | run crawlers to get proxy |
| 33 | :return: |
| 34 | """ |
| 35 | if self.is_full(): |
| 36 | return |
| 37 | for crawler in self.crawlers: |
| 38 | logger.info(f'crawler {crawler} to get proxy') |
| 39 | for proxy in crawler.crawl(): |
| 40 | self.redis.add(proxy) |
| 41 | [self.redis.add(proxy, redis_key=tester.key) for tester in self.testers] |
| 42 | |
| 43 | |
| 44 | if __name__ == '__main__': |
no outgoing calls
no test coverage detected