(self, input)
| 27 | |
| 28 | # input: the question line. e.g. "Question: What is the capital of France?" |
| 29 | def run(self, input): |
| 30 | # run is stateless, so we need to reset the evidences |
| 31 | self._reinitialize() |
| 32 | result = {} |
| 33 | st = time.time() |
| 34 | # Plan |
| 35 | planner_response = self.planner.run(input, log=True) |
| 36 | plan = planner_response["output"] |
| 37 | planner_log = planner_response["input"] + planner_response["output"] |
| 38 | self.plans = self._parse_plans(plan) |
| 39 | self.planner_evidences = self._parse_planner_evidences(plan) |
| 40 | #assert len(self.plans) == len(self.planner_evidences) |
| 41 | |
| 42 | # Work |
| 43 | self._get_worker_evidences() |
| 44 | worker_log = "" |
| 45 | for i in range(len(self.plans)): |
| 46 | e = f"#E{i + 1}" |
| 47 | worker_log += f"{self.plans[i]}\nEvidence:\n{self.worker_evidences[e]}\n" |
| 48 | |
| 49 | # Solve |
| 50 | solver_response = self.solver.run(input, worker_log, log=True) |
| 51 | output = solver_response["output"] |
| 52 | solver_log = solver_response["input"] + solver_response["output"] |
| 53 | |
| 54 | result["wall_time"] = time.time() - st |
| 55 | result["input"] = input |
| 56 | result["output"] = output |
| 57 | result["planner_log"] = planner_log |
| 58 | result["worker_log"] = worker_log |
| 59 | result["solver_log"] = solver_log |
| 60 | result["tool_usage"] = self.tool_counter |
| 61 | result["steps"] = len(self.plans) + 1 |
| 62 | result["total_tokens"] = planner_response["prompt_tokens"] + planner_response["completion_tokens"] \ |
| 63 | + solver_response["prompt_tokens"] + solver_response["completion_tokens"] \ |
| 64 | + self.tool_counter.get("LLM_token", 0) \ |
| 65 | + self.tool_counter.get("Calculator_token", 0) |
| 66 | result["token_cost"] = self.planner_token_unit_price * (planner_response["prompt_tokens"] + planner_response["completion_tokens"]) \ |
| 67 | + self.solver_token_unit_price * (solver_response["prompt_tokens"] + solver_response["completion_tokens"]) \ |
| 68 | + self.tool_token_unit_price * (self.tool_counter.get("LLM_token", 0) + self.tool_counter.get("Calculator_token", 0)) |
| 69 | result["tool_cost"] = self.tool_counter.get("Google", 0) * self.google_unit_price |
| 70 | result["total_cost"] = result["token_cost"] + result["tool_cost"] |
| 71 | |
| 72 | return result |
| 73 | |
| 74 | def _parse_plans(self, response): |
| 75 | plans = [] |
no test coverage detected