()
| 12 | |
| 13 | |
| 14 | async def main() -> None: |
| 15 | # Open request queues for both crawlers with different aliases |
| 16 | playwright_rq = await RequestQueue.open(alias='playwright-requests') |
| 17 | parsel_rq = await RequestQueue.open(alias='parsel-requests') |
| 18 | |
| 19 | # Use a shared session pool between both crawlers |
| 20 | async with SessionPool() as session_pool: |
| 21 | playwright_crawler = PlaywrightCrawler( |
| 22 | # Set the request queue for Playwright crawler |
| 23 | request_manager=playwright_rq, |
| 24 | session_pool=session_pool, |
| 25 | # Configure concurrency settings for Playwright crawler |
| 26 | concurrency_settings=ConcurrencySettings( |
| 27 | max_concurrency=5, desired_concurrency=5 |
| 28 | ), |
| 29 | # Set `keep_alive`` so that the crawler does not stop working when there are |
| 30 | # no requests in the queue. |
| 31 | keep_alive=True, |
| 32 | ) |
| 33 | |
| 34 | parsel_crawler = ParselCrawler( |
| 35 | # Set the request queue for Parsel crawler |
| 36 | request_manager=parsel_rq, |
| 37 | session_pool=session_pool, |
| 38 | # Configure concurrency settings for Parsel crawler |
| 39 | concurrency_settings=ConcurrencySettings( |
| 40 | max_concurrency=10, desired_concurrency=10 |
| 41 | ), |
| 42 | # Set maximum requests per crawl for Parsel crawler |
| 43 | max_requests_per_crawl=50, |
| 44 | ) |
| 45 | |
| 46 | @playwright_crawler.router.default_handler |
| 47 | async def handle_playwright(context: PlaywrightCrawlingContext) -> None: |
| 48 | context.log.info(f'Playwright Processing {context.request.url}...') |
| 49 | |
| 50 | title = await context.page.title() |
| 51 | # Push the extracted data to the dataset for Playwright crawler |
| 52 | await context.push_data( |
| 53 | {'title': title, 'url': context.request.url, 'source': 'playwright'}, |
| 54 | dataset_name='playwright-data', |
| 55 | ) |
| 56 | |
| 57 | @parsel_crawler.router.default_handler |
| 58 | async def handle_parsel(context: ParselCrawlingContext) -> None: |
| 59 | context.log.info(f'Parsel Processing {context.request.url}...') |
| 60 | |
| 61 | title = context.parsed_content.css('title::text').get() |
| 62 | # Push the extracted data to the dataset for Parsel crawler |
| 63 | await context.push_data( |
| 64 | {'title': title, 'url': context.request.url, 'source': 'parsel'}, |
| 65 | dataset_name='parsel-data', |
| 66 | ) |
| 67 | |
| 68 | # Enqueue links to the Playwright request queue for blog pages |
| 69 | await context.enqueue_links( |
| 70 | selector='a[href*="/blog/"]', rq_alias='playwright-requests' |
| 71 | ) |
no test coverage detected