Method execute

scrapegraphai/nodes/parse_node.py:62–129

Executes the node's logic to parse the HTML document content and split it into chunks. Args: state (dict): The current state of the graph. The input keys will be used to fetch the correct data from the state. Returns: dic

(self, state: dict)

        self.chunk_size = node_config.get("chunk_size")

    def execute(self, state: dict) -> dict:
        """
        Executes the node's logic to parse the HTML document content and split it into chunks.

        Args:
            state (dict): The current state of the graph. The input keys will be used to fetch the
                            correct data from the state.

        Returns:
            dict: The updated state with the output key containing the parsed content chunks.

        Raises:
            KeyError: If the input keys are not found in the state, indicating that the
                        necessary information for parsing the content is missing.
        """

        self.logger.info(f"--- Executing {self.node_name} Node ---")

        input_keys = self.get_input_keys(state)
        input_data = [state[key] for key in input_keys]
        docs_transformed = input_data[0]
        source = input_data[1] if self.parse_urls else None

        if self.parse_html:
            docs_transformed = Html2TextTransformer(
                ignore_links=False
            ).transform_documents(input_data[0])
            docs_transformed = docs_transformed[0]

            link_urls, img_urls = self._extract_urls(
                docs_transformed.page_content, source
            )

            chunks = split_text_into_chunks(
                text=docs_transformed.page_content,
                chunk_size=self.chunk_size - 250,
            )
        else:
            docs_transformed = docs_transformed[0]

            try:
                link_urls, img_urls = self._extract_urls(
                    docs_transformed.page_content, source
                )
            except Exception:
                link_urls, img_urls = "", ""

            chunk_size = self.chunk_size
            chunk_size = min(chunk_size - 500, int(chunk_size * 0.8))

            if isinstance(docs_transformed, Document):
                chunks = split_text_into_chunks(
                    text=docs_transformed.page_content,
                    chunk_size=chunk_size,
                )
            else:
                chunks = split_text_into_chunks(
                    text=docs_transformed, chunk_size=chunk_size