:param usage: one of SCORE_MAPS's keys, such as https :param length: if total available proxies are less than length, you must refresh pool :param strategy: the load balance of proxy ip, the value is one of ['robin', 'greedy'] :param fast_response: if
(self, usage, strategy='robin', length=10,
fast_response=5, redis_args=None)
| 52 | |
| 53 | class ProxyFetcher: |
| 54 | def __init__(self, usage, strategy='robin', length=10, |
| 55 | fast_response=5, redis_args=None): |
| 56 | """ |
| 57 | :param usage: one of SCORE_MAPS's keys, such as https |
| 58 | :param length: if total available proxies are less than length, |
| 59 | you must refresh pool |
| 60 | :param strategy: the load balance of proxy ip, the value is |
| 61 | one of ['robin', 'greedy'] |
| 62 | :param fast_response: if you use greedy strategy, if will be needed to |
| 63 | decide whether a proxy ip should continue to be used |
| 64 | :param redis_args: redis connetion args, it's a dict, the keys include host, port, db and password |
| 65 | """ |
| 66 | self.score_queue = SCORE_MAPS.get(usage) |
| 67 | self.ttl_queue = TTL_MAPS.get(usage) |
| 68 | self.speed_queue = SPEED_MAPS.get(usage) |
| 69 | self.strategy = strategy |
| 70 | # pool is a queue, which is FIFO |
| 71 | self.pool = list() |
| 72 | self.length = length |
| 73 | self.fast_response = fast_response |
| 74 | self.handlers = [RobinStrategy(), GreedyStrategy()] |
| 75 | if isinstance(redis_args, dict): |
| 76 | self.conn = get_redis_conn(**redis_args) |
| 77 | else: |
| 78 | self.conn = get_redis_conn() |
| 79 | |
| 80 | def get_proxy(self): |
| 81 | """ |
no test coverage detected