Method parse_into_text

docreader/parser/web_parser.py:206–258

Parse web page content into a Document object. Args: content: URL encoded as bytes Returns: Document object containing the parsed markdown content

(self, content: bytes)

            return empty

    def parse_into_text(self, content: bytes) -> Document:
        """Parse web page content into a Document object.

        Args:
            content: URL encoded as bytes

        Returns:
            Document object containing the parsed markdown content
        """
        url = endecode.decode_bytes(content)

        logger.info(f"Scraping web page: {url}")
        scrape_result = asyncio.run(self.scrape(url))
        if not scrape_result.html and not scrape_result.visible_text:
            logger.error("Failed to scrape web page (no HTML or visible text)")
            return Document(content=f"Error parsing web page: {url}")

        md_text = extract_markdown_from_html(scrape_result.html)
        if not md_text:
            md_text = build_visible_text_fallback(
                scrape_result.visible_text,
                scrape_result.page_title,
            )
            if md_text:
                logger.info(
                    "Trafilatura empty; using Playwright visible-text fallback (%d chars)",
                    len(md_text),
                )

        if not md_text:
            logger.error("Failed to parse web page")
            return Document(content=f"Error parsing web page: {url}")

        metadata = {}
        title_match = re.search(r"^title:\s*(.+)", md_text, re.MULTILINE)
        if title_match:
            extracted_title = title_match.group(1).strip()
            if extracted_title:
                metadata["title"] = extracted_title
                logger.info(
                    f"Extracted article title from trafilatura: {extracted_title}"
                )
        elif scrape_result.page_title:
            metadata["title"] = scrape_result.page_title.strip()
            logger.info(
                "Using page title from Playwright: %s", metadata["title"]
            )
        else:
            logger.info(
                "No title found in trafilatura output, first 200 chars: %r",
                md_text[:200],
            )
        return Document(content=md_text, metadata=metadata)


class WebParser(PipelineParser):