Class OpenDataLoaderParser

docreader/parser/opendataloader_parser.py:291–360

Parse PDFs with OpenDataLoader (layout-aware markdown + external images).

class OpenDataLoaderParser(BaseParser):
    """Parse PDFs with OpenDataLoader (layout-aware markdown + external images)."""

    def __init__(self, *args: Any, **kwargs: Any):
        self._engine_overrides: Dict[str, Any] = {
            k: v
            for k, v in kwargs.items()
            if k.startswith("odl_") or k in ("mineru_endpoint", "mineru_api_key")
        }
        super().__init__(*args, **kwargs)

    def parse_into_text(self, content: bytes) -> Document:
        ok, msg = opendataloader_available(self._engine_overrides)
        if not ok:
            raise RuntimeError(msg)

        safe_name = os.path.basename(self.file_name) or "document.pdf"
        if not safe_name.lower().endswith(".pdf"):
            safe_name = f"{os.path.splitext(safe_name)[0] or 'document'}.pdf"
        pdf_stem = os.path.splitext(safe_name)[0]

        max_workers = CONFIG.odl_max_workers
        with parser_worker_limit("opendataloader", max_workers):
            with tempfile.TemporaryDirectory(prefix="weknora-odl-") as tmp_dir:
                pdf_path = os.path.join(tmp_dir, safe_name)
                with open(pdf_path, "wb") as f:
                    f.write(content)
                image_dir = os.path.join(tmp_dir, "images")
                os.makedirs(image_dir, exist_ok=True)

                _run_convert(
                    pdf_path,
                    tmp_dir,
                    image_dir,
                    overrides=self._engine_overrides,
                )

                md_path = _find_markdown_file(tmp_dir, pdf_stem)
                with open(md_path, encoding="utf-8", errors="replace") as f:
                    text = f.read()

                images = _collect_images_under_output(tmp_dir)
                text = _rewrite_markdown_image_refs(text, images)

        if len(text.strip()) < _MIN_CHARS_PER_PAGE:
            logger.info(
                "OpenDataLoaderParser: %s yielded little text; "
                "falling back to builtin scanned renderer",
                self.file_name,
            )
            from docreader.parser.pdf_parser import PDFScannedParser

            return PDFScannedParser(
                file_name=self.file_name, file_type=self.file_type
            ).parse_into_text(content)

        logger.info(
            "OpenDataLoaderParser: %s -> content_len=%d images=%d",