()
| 36 | |
| 37 | |
| 38 | async def main() -> None: |
| 39 | # Prepare data mapping for hosts |
| 40 | apify_host = URL('https://apify.com/sitemap.xml').host |
| 41 | crawlee_host = URL('https://crawlee.dev/sitemap.xml').host |
| 42 | |
| 43 | if not apify_host or not crawlee_host: |
| 44 | raise ValueError('Unable to extract host from URLs') |
| 45 | |
| 46 | data_map = { |
| 47 | apify_host: { |
| 48 | 'label': 'apify', |
| 49 | 'user_data': {'source': 'apify'}, |
| 50 | }, |
| 51 | crawlee_host: { |
| 52 | 'label': 'crawlee', |
| 53 | 'user_data': {'source': 'crawlee'}, |
| 54 | }, |
| 55 | } |
| 56 | |
| 57 | # Initialize the SitemapRequestLoader with the transform function |
| 58 | async with SitemapRequestLoader( |
| 59 | # Set the sitemap URLs and the HTTP client |
| 60 | sitemap_urls=['https://crawlee.dev/sitemap.xml', 'https://apify.com/sitemap.xml'], |
| 61 | http_client=ImpitHttpClient(), |
| 62 | transform_request_function=create_transform_request(data_map), |
| 63 | ) as sitemap_loader: |
| 64 | # Convert the sitemap loader to a request manager |
| 65 | request_manager = await sitemap_loader.to_tandem() |
| 66 | |
| 67 | # Create and configure the crawler |
| 68 | crawler = BeautifulSoupCrawler( |
| 69 | request_manager=request_manager, |
| 70 | max_requests_per_crawl=10, |
| 71 | ) |
| 72 | |
| 73 | # Create default handler for requests without a specific label |
| 74 | @crawler.router.default_handler |
| 75 | async def handler(context: BeautifulSoupCrawlingContext) -> None: |
| 76 | source = context.request.user_data.get('source', 'unknown') |
| 77 | context.log.info( |
| 78 | f'Processing request: {context.request.url} from source: {source}' |
| 79 | ) |
| 80 | |
| 81 | # Create handler for requests labeled 'apify' |
| 82 | @crawler.router.handler('apify') |
| 83 | async def apify_handler(context: BeautifulSoupCrawlingContext) -> None: |
| 84 | source = context.request.user_data.get('source', 'unknown') |
| 85 | context.log.info( |
| 86 | f'Apify handler processing: {context.request.url} from source: {source}' |
| 87 | ) |
| 88 | |
| 89 | # Create handler for requests labeled 'crawlee' |
| 90 | @crawler.router.handler('crawlee') |
| 91 | async def crawlee_handler(context: BeautifulSoupCrawlingContext) -> None: |
| 92 | source = context.request.user_data.get('source', 'unknown') |
| 93 | context.log.info( |
| 94 | f'Crawlee handler processing: {context.request.url} from source: {source}' |
| 95 | ) |
no test coverage detected