redis connection client of proxypool
| 14 | |
| 15 | |
| 16 | class RedisClient(object): |
| 17 | """ |
| 18 | redis connection client of proxypool |
| 19 | """ |
| 20 | |
| 21 | def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, |
| 22 | connection_string=REDIS_CONNECTION_STRING, **kwargs): |
| 23 | """ |
| 24 | init redis client |
| 25 | :param host: redis host |
| 26 | :param port: redis port |
| 27 | :param password: redis password |
| 28 | :param connection_string: redis connection_string |
| 29 | """ |
| 30 | # if set connection_string, just use it |
| 31 | if connection_string: |
| 32 | self.db = redis.StrictRedis.from_url(connection_string, decode_responses=True, **kwargs) |
| 33 | else: |
| 34 | self.db = redis.StrictRedis( |
| 35 | host=host, port=port, password=password, db=db, decode_responses=True, **kwargs) |
| 36 | |
| 37 | def add(self, proxy: Proxy, score=PROXY_SCORE_INIT, redis_key=REDIS_KEY) -> int: |
| 38 | """ |
| 39 | add proxy and set it to init score |
| 40 | :param proxy: proxy, ip:port, like 8.8.8.8:88 |
| 41 | :param score: int score |
| 42 | :return: result |
| 43 | """ |
| 44 | if not is_valid_proxy(f'{proxy.host}:{proxy.port}'): |
| 45 | logger.info(f'invalid proxy {proxy}, throw it') |
| 46 | return |
| 47 | if not self.exists(proxy, redis_key): |
| 48 | if IS_REDIS_VERSION_2: |
| 49 | return self.db.zadd(redis_key, score, proxy.string()) |
| 50 | return self.db.zadd(redis_key, {proxy.string(): score}) |
| 51 | |
| 52 | def random(self, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> Proxy: |
| 53 | """ |
| 54 | get random proxy |
| 55 | firstly try to get proxy with max score |
| 56 | if not exists, try to get proxy by rank |
| 57 | if not exists, raise error |
| 58 | :return: proxy, like 8.8.8.8:8 |
| 59 | """ |
| 60 | # try to get proxy with max score |
| 61 | proxies = self.db.zrangebyscore( |
| 62 | redis_key, proxy_score_max, proxy_score_max) |
| 63 | if len(proxies): |
| 64 | return convert_proxy_or_proxies(choice(proxies)) |
| 65 | # else get proxy by rank |
| 66 | proxies = self.db.zrevrange( |
| 67 | redis_key, proxy_score_min, proxy_score_max) |
| 68 | if len(proxies): |
| 69 | return convert_proxy_or_proxies(choice(proxies)) |
| 70 | # else raise error |
| 71 | raise PoolEmptyException |
| 72 | |
| 73 | def randoms(self, count, redis_key=REDIS_KEY, proxy_score_min=PROXY_SCORE_MIN, proxy_score_max=PROXY_SCORE_MAX) -> List[Proxy]: |