Method reset_proxy_pool

feapder/network/proxy_pool_old.py:601–655

重置代理池 :param force: 是否强制重置代理池 :return:

(self, force: bool = False)

        return self.proxy_queue.put_nowait(proxy_item)

    def reset_proxy_pool(self, force: bool = False):
        """
        重置代理池
        :param force: 是否强制重置代理池
        :return:
        """
        if not self.reset_lock:
            # 必须用时调用 否则 可能存在 gevent patch前 threading就已经被导入 导致的Rlock patch失效
            import threading

            self.reset_lock = threading.RLock()
        with self.reset_lock:
            if (
                force
                or self.proxy_queue is None
                or (
                    self.max_queue_size > 0
                    and self.proxy_queue.qsize() < self.max_queue_size / 2
                )
                or (
                    self.max_queue_size < 0
                    and self.proxy_queue.qsize() < self.real_max_proxy_count / 2
                )
                or self.no_valid_proxy_times >= 5
            ):
                if time.time() - self.last_reset_time < self.reset_interval:
                    self.reset_fast_count += 1
                    if self.reset_fast_count % 10 == 0:
                        self.logger.debug(
                            "代理池重置的太快了:) {}".format(self.reset_fast_count)
                        )
                        time.sleep(1)
                else:
                    self.clear()
                    if self.proxy_queue is None:
                        import queue

                        self.proxy_queue = queue.Queue()
                    # TODO 这里获取到的可能重复
                    proxies_list = get_proxy_from_url(**self.kwargs)
                    self.real_max_proxy_count = len(proxies_list)
                    if 0 < self.max_queue_size < self.real_max_proxy_count:
                        proxies_list = random.sample(proxies_list, self.max_queue_size)
                    _valid_count = self.append_proxies(proxies_list)
                    self.last_reset_time = time.time()
                    self.no_valid_proxy_times = 0
                    self.logger.debug(
                        "重置代理池成功: 获取{}, 成功添加{}, 失效{},  当前代理数{},".format(
                            len(proxies_list),
                            _valid_count,
                            len(self.invalid_proxy_dict),
                            len(self.proxy_dict),
                        )
                    )
        return

    def tag_proxy(self, proxies_list: list, flag: int, *, delay=30) -> bool:
        """