Initialize a new instance. Args: plugins: Browser plugins serve as wrappers around various browser automation libraries, providing a consistent interface across different libraries. operation_timeout: Operations of the underlying automation libraries,
(
self,
plugins: Sequence[BrowserPlugin] | None = None,
*,
operation_timeout: timedelta = timedelta(seconds=60),
browser_inactive_threshold: timedelta = timedelta(seconds=10),
identify_inactive_browsers_interval: timedelta = timedelta(seconds=20),
close_inactive_browsers_interval: timedelta = timedelta(seconds=30),
retire_browser_after_page_count: int = 100,
)
| 46 | """The length of the newly generated page ID.""" |
| 47 | |
| 48 | def __init__( |
| 49 | self, |
| 50 | plugins: Sequence[BrowserPlugin] | None = None, |
| 51 | *, |
| 52 | operation_timeout: timedelta = timedelta(seconds=60), |
| 53 | browser_inactive_threshold: timedelta = timedelta(seconds=10), |
| 54 | identify_inactive_browsers_interval: timedelta = timedelta(seconds=20), |
| 55 | close_inactive_browsers_interval: timedelta = timedelta(seconds=30), |
| 56 | retire_browser_after_page_count: int = 100, |
| 57 | ) -> None: |
| 58 | """Initialize a new instance. |
| 59 | |
| 60 | Args: |
| 61 | plugins: Browser plugins serve as wrappers around various browser automation libraries, |
| 62 | providing a consistent interface across different libraries. |
| 63 | operation_timeout: Operations of the underlying automation libraries, such as launching a browser |
| 64 | or opening a new page, can sometimes get stuck. To prevent `BrowserPool` from becoming unresponsive, |
| 65 | we add a timeout to these operations. |
| 66 | browser_inactive_threshold: The period of inactivity after which a browser is considered as inactive. |
| 67 | identify_inactive_browsers_interval: The period of inactivity after which a browser is considered |
| 68 | as retired. |
| 69 | close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers |
| 70 | and closes them. The browser is considered as inactive if it has no active pages and has been idle |
| 71 | for the specified period. The browser is considered as retired if it has no active pages and has total |
| 72 | pages count greater than or equal to `retire_browser_after_page_count`. |
| 73 | retire_browser_after_page_count: The maximum number of processed pages after which the browser is considered |
| 74 | as retired. |
| 75 | """ |
| 76 | self._plugins = plugins or [PlaywrightBrowserPlugin()] |
| 77 | self._operation_timeout = operation_timeout |
| 78 | self._browser_inactive_threshold = browser_inactive_threshold |
| 79 | |
| 80 | self._active_browsers = list[BrowserController]() |
| 81 | """A list of browsers currently active and being used to open pages.""" |
| 82 | |
| 83 | self._inactive_browsers = list[BrowserController]() |
| 84 | """A list of browsers currently inactive and not being used to open new pages, |
| 85 | but may still contain open pages.""" |
| 86 | |
| 87 | self._identify_inactive_browsers_task = RecurringTask( |
| 88 | self._identify_inactive_browsers, |
| 89 | identify_inactive_browsers_interval, |
| 90 | ) |
| 91 | |
| 92 | self._close_inactive_browsers_task = RecurringTask( |
| 93 | self._close_inactive_browsers, |
| 94 | close_inactive_browsers_interval, |
| 95 | ) |
| 96 | |
| 97 | self._total_pages_count = 0 |
| 98 | self._retire_browser_after_page_count = retire_browser_after_page_count |
| 99 | self._pages = WeakValueDictionary[str, CrawleePage]() # Track the pages in the pool |
| 100 | self._plugins_cycle = itertools.cycle(self._plugins) # Cycle through the plugins |
| 101 | |
| 102 | # Hooks for custom behavior at different stages of the browser and page lifecycles. |
| 103 | self._pre_launch_hooks: list[Callable[[str, BrowserPlugin], Awaitable[None]]] = [] |
| 104 | self._post_launch_hooks: list[Callable[[str, BrowserController], Awaitable[None]]] = [] |
| 105 | self._pre_page_create_hooks: list[ |
nothing calls this directly
no test coverage detected