Method execute

scrapegraphai/nodes/fetch_node.py:93–126

Executes the node's logic to fetch HTML content from a specified URL and update the state with this content.

(self, state)

        )

    def execute(self, state):
        """
        Executes the node's logic to fetch HTML content from a specified URL and
        update the state with this content.
        """
        self.logger.info(f"--- Executing {self.node_name} Node ---")

        input_keys = self.get_input_keys(state)
        input_data = [state[key] for key in input_keys]

        source = input_data[0]
        input_type = input_keys[0]

        handlers = {
            "json_dir": self.handle_directory,
            "xml_dir": self.handle_directory,
            "csv_dir": self.handle_directory,
            "pdf_dir": self.handle_directory,
            "md_dir": self.handle_directory,
            "pdf": self.handle_file,
            "csv": self.handle_file,
            "json": self.handle_file,
            "xml": self.handle_file,
            "md": self.handle_file,
        }

        if input_type in handlers:
            return handlers[input_type](state, input_type, source)
        elif input_type == "local_dir":
            return self.handle_local_source(state, source)
        elif input_type == "url":
            return self.handle_web_source(state, source)
        else:
            raise ValueError(f"Invalid input type: {input_type}")

    def handle_directory(self, state, input_type, source):
        """