| 18 | |
| 19 | @Singleton |
| 20 | class WebDriverPool: |
| 21 | def __init__( |
| 22 | self, pool_size=5, driver_cls=SeleniumDriver, thread_safe=False, **kwargs |
| 23 | ): |
| 24 | """ |
| 25 | |
| 26 | Args: |
| 27 | pool_size: driver池的大小 |
| 28 | driver: 驱动类型 |
| 29 | thread_safe: 是否线程安全 |
| 30 | 是则每个线程拥有一个driver，pool_size无效，driver数量为线程数 |
| 31 | 否则每个线程从池中获取driver |
| 32 | **kwargs: |
| 33 | """ |
| 34 | self.pool_size = pool_size |
| 35 | self.driver_cls = driver_cls |
| 36 | self.thread_safe = thread_safe |
| 37 | self.kwargs = kwargs |
| 38 | |
| 39 | self.queue = queue.Queue(maxsize=pool_size) |
| 40 | self.lock = threading.RLock() |
| 41 | self.driver_count = 0 |
| 42 | self.ctx = threading.local() |
| 43 | |
| 44 | @property |
| 45 | def driver(self): |
| 46 | if not hasattr(self.ctx, "driver"): |
| 47 | self.ctx.driver = None |
| 48 | return self.ctx.driver |
| 49 | |
| 50 | @driver.setter |
| 51 | def driver(self, driver): |
| 52 | self.ctx.driver = driver |
| 53 | |
| 54 | @property |
| 55 | def is_full(self): |
| 56 | return self.driver_count >= self.pool_size |
| 57 | |
| 58 | def create_driver(self, user_agent: str = None, proxy: str = None): |
| 59 | kwargs = self.kwargs.copy() |
| 60 | if user_agent: |
| 61 | kwargs["user_agent"] = user_agent |
| 62 | if proxy: |
| 63 | kwargs["proxy"] = proxy |
| 64 | return self.driver_cls(**kwargs) |
| 65 | |
| 66 | def get(self, user_agent: str = None, proxy: str = None): |
| 67 | """ |
| 68 | 获取webdriver |
| 69 | 当webdriver为新实例时会使用 user_agen, proxy, cookie参数来创建 |
| 70 | Args: |
| 71 | user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 |
| 72 | proxy: xxx.xxx.xxx.xxx |
| 73 | Returns: |
| 74 | |
| 75 | """ |
| 76 | if not self.is_full and not self.thread_safe: |
| 77 | with self.lock: |
no outgoing calls