Method request_parse

module/stepback_crawler.py:105–136

A safe and reliable call to LLMs, which confirm that the output can be parsed by json.loads(). Args: query (str): the query to prompt the LLM html (str): the HTML text for Returns: str: a dict parsed from the output of LLM

(self, 
                      query: str,
                      keys: list[str] = [])

        self.error_max_times = error_max_times

    def request_parse(self, 
                      query: str,
                      keys: list[str] = []) -> dict[str, str]:
        """A safe and reliable call to LLMs, which confirm that the output can be parsed by json.loads().

        Args:
            query (str): the query to prompt the LLM
            html (str): the HTML text for 

        Returns:
            str: a dict parsed from the output of LLM
        """
        pattern = r'\{.*?\}'
        target = False
        for _ in range(self.error_max_times):
            response = self.api(query)
            matches = re.findall(pattern, response, re.DOTALL)
            try:
                for match in matches:
                    res = json.loads(match) # type: ignore
                    for key in keys:
                        assert res[key]
                    target = True
                if target:
                    break
            except:
                pass
        if target:
            #print(res)
            return res
        else:
            return {key:"" for key in keys}
        
    def generate_sequence_html(self,
                          instruction: str,