Process a raw content sitemap source.
(
source: SitemapSource,
depth: int,
visited_sitemap_urls: set[str],
sources: list[SitemapSource],
*,
emit_nested_sitemaps: bool,
enqueue_strategy: EnqueueStrategy,
)
| 285 | |
| 286 | |
| 287 | async def _process_raw_source( |
| 288 | source: SitemapSource, |
| 289 | depth: int, |
| 290 | visited_sitemap_urls: set[str], |
| 291 | sources: list[SitemapSource], |
| 292 | *, |
| 293 | emit_nested_sitemaps: bool, |
| 294 | enqueue_strategy: EnqueueStrategy, |
| 295 | ) -> AsyncGenerator[SitemapUrl | NestedSitemap, None]: |
| 296 | """Process a raw content sitemap source.""" |
| 297 | if 'content' not in source: |
| 298 | logger.warning(f'Raw source missing content: {source}') |
| 299 | return |
| 300 | |
| 301 | content = source['content'] |
| 302 | parser = _get_parser('text/xml') |
| 303 | |
| 304 | try: |
| 305 | # Process the content |
| 306 | async for item in parser.process_chunk(content): |
| 307 | async for result in _process_sitemap_item( |
| 308 | item, |
| 309 | source, |
| 310 | depth, |
| 311 | visited_sitemap_urls, |
| 312 | sources, |
| 313 | emit_nested_sitemaps=emit_nested_sitemaps, |
| 314 | enqueue_strategy=enqueue_strategy, |
| 315 | ): |
| 316 | if result: |
| 317 | yield result |
| 318 | |
| 319 | # Process any remaining content |
| 320 | async for item in parser.flush(): |
| 321 | async for result in _process_sitemap_item( |
| 322 | item, |
| 323 | source, |
| 324 | depth, |
| 325 | visited_sitemap_urls, |
| 326 | sources, |
| 327 | emit_nested_sitemaps=emit_nested_sitemaps, |
| 328 | enqueue_strategy=enqueue_strategy, |
| 329 | ): |
| 330 | if result: |
| 331 | yield result |
| 332 | except Exception as e: |
| 333 | logger.warning(f'Failed to parse raw sitemap content: {e}') |
| 334 | finally: |
| 335 | parser.close() |
| 336 | |
| 337 | |
| 338 | async def _fetch_and_process_sitemap( |
no test coverage detected