Method node_eval

src/agents/optimization/utils.py:155–229

Evaluates the node and saves the result to the file. Args: case (Case): The case to be evaluated. solution (Solution): The solution configuration. llm (OpenAILLM): The large language model instance. logger (logging.Logger): Logger for

(case: Case, solution: Solution, llm: OpenAILLM, logger)

    @staticmethod
    def node_eval(case: Case, solution: Solution, llm: OpenAILLM, logger):
        """
        Evaluates the node and saves the result to the file.

        Args:
            case (Case): The case to be evaluated.
            solution (Solution): The solution configuration.
            llm (OpenAILLM): The large language model instance.
            logger (logging.Logger): Logger for logging information and errors.
        """
        if logger:
            logger.debug("Start Node evaluation, case id is " + case.case_id)
        assert case.trajectory is not None and len(case.trajectory.states) > 0, \
            "case.trajectory is None or len is 0, please forward the case first"
        input_dict = {}

        cur_node_name = None
        start_idx = 0
        previous_node_summary = "You are the first node."
        for idx, state in enumerate(case.trajectory.states):
            # If the node_name changes, it's a new node
            if state.node.node_name != cur_node_name:
                start_idx = idx
                cur_node_name = state.node.node_name

            # If it's the last state of the current node, then it needs to be evaluated
            if (idx == len(case.trajectory.states) - 1
                    or case.trajectory.states[idx + 1].node.node_name != cur_node_name):
                # Obtain the output of all roles of the current node
                role_chat = ""
                for i in range(start_idx, idx + 1):
                    role_chat += (case.trajectory.states[i].action.agent_role + " : "
                                  + case.trajectory.states[i].action.content + "\n")
                input_dict["role_chat"] = role_chat
                input_dict["node_config"] = {
                    "node_name": solution.sop.nodes[cur_node_name].node_name,
                    "node_description": solution.sop.nodes[cur_node_name].node_description,
                }
                input_dict["previous_node_summary"] = previous_node_summary

                eval_prompt = eval_node_prompt.format(**input_dict)
                response, content = llm.get_response(
                    chat_messages=None,
                    system_prompt="",
                    last_prompt=eval_prompt,
                    stream=False,
                )

                # To extract the result, first extract the part of the result package, then convert it to json,
                # and input it into the state, specifically a summary and an evaluation
                eval_result = OptimUtils.extract_data_from_response(content, ["result"])
                try:
                    eval_result = json.loads(eval_result["result"])
                except:
                    logger.error(
                        "Error in node evaluation, case id is " + case.case_id
                        + ", the result is not a legal json string, the result is: " + content
                    )