Parse content from a URL to markdown.
(
self,
url: str,
title: str,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
)
| 63 | return result |
| 64 | |
| 65 | def parse_url( |
| 66 | self, |
| 67 | url: str, |
| 68 | title: str, |
| 69 | parser_engine: Optional[str] = None, |
| 70 | engine_overrides: Optional[dict[str, Any]] = None, |
| 71 | ) -> Document: |
| 72 | """Parse content from a URL to markdown.""" |
| 73 | logger.info("Parsing URL: %s, title: %s", url, title) |
| 74 | |
| 75 | parser = WebParser(title=title) |
| 76 | logger.info("Starting to parse URL content") |
| 77 | result = parser.parse(url.encode()) |
| 78 | |
| 79 | if not result.content: |
| 80 | logger.warning("Parser returned empty content for url: %s", url) |
| 81 | logger.info("Parsed url %s, content length=%d", url, len(result.content)) |
| 82 | return result |
no test coverage detected