Method convert

docling/backend/html_backend.py:481–555

(self)

    @override
    def convert(self) -> DoclingDocument:
        _log.debug("Starting HTML conversion...")
        if not self.is_valid():
            raise RuntimeError("Invalid HTML document.")

        origin = DocumentOrigin(
            filename=self.file.name or "file",
            mimetype="text/html",
            binary_hash=self.document_hash,
        )
        doc = DoclingDocument(name=self.file.stem or "file", origin=origin)

        if cast(HTMLBackendOptions, self.options).render_page:
            self._render_with_browser()
            if self._rendered_html:
                self.soup = BeautifulSoup(self._rendered_html, "html.parser")

        if self._rendered_page_images and self._rendered_page_size:
            render_dpi = cast(HTMLBackendOptions, self.options).render_dpi
            for page_no, page_image in enumerate(self._rendered_page_images, start=1):
                doc.add_page(
                    page_no=page_no,
                    size=self._rendered_page_size,
                    image=ImageRef.from_pil(image=page_image, dpi=render_dpi),
                )

        assert self.soup is not None
        # set the title as furniture, since it is part of the document metadata
        title = self.soup.title
        if title and self.options.add_title:
            title_text = title.get_text(separator=" ", strip=True)
            title_clean = HTMLDocumentBackend._clean_unicode(title_text)
            doc.add_title(
                text=title_clean,
                orig=title_text,
                content_layer=ContentLayer.FURNITURE,
            )
        # remove script and style tags
        for tag in self.soup(["script", "noscript", "style"]):
            tag.decompose()
        # remove any hidden tag
        for tag in self.soup(hidden=True):
            tag.decompose()
        # fix flow content that is not permitted inside <p>
        HTMLDocumentBackend._fix_invalid_paragraph_structure(self.soup)

        content = self.soup.body or self.soup

        # normalize <br> tags - use sentinel to distinguish from source newlines
        for text_node in content.find_all(string=True):
            if _BR_SENTINEL in text_node:
                text_node.replace_with(text_node.replace(_BR_SENTINEL, ""))
        for br in content("br"):
            br.replace_with(NavigableString(_BR_SENTINEL))

        # Furniture before the first heading rule, except for headers in tables
        header = None
        # Find all headers first