| 47 | |
| 48 | |
| 49 | class TrajectoryAnalyzer: |
| 50 | def __init__(self, traj_path: str): |
| 51 | try: |
| 52 | with open(traj_path) as f: |
| 53 | self.traj = json.load(f) |
| 54 | self.messages = self.traj.get("messages", []) |
| 55 | except (json.JSONDecodeError, FileNotFoundError, KeyError): |
| 56 | self.traj = {} |
| 57 | self.messages = [] |
| 58 | |
| 59 | @property |
| 60 | def steps(self) -> int: |
| 61 | return sum([1 for x in self.traj["messages"] if x["role"] == "assistant"]) |
| 62 | |
| 63 | @property |
| 64 | def failure_stats(self) -> dict: |
| 65 | failed_commands = 0 |
| 66 | failed_command_types = {} |
| 67 | for i, message in enumerate(self.messages): |
| 68 | if message["role"] == "user": |
| 69 | content = message.get("content", "") |
| 70 | |
| 71 | # Handle both list and string content formats |
| 72 | if isinstance(content, list) and content: |
| 73 | text_content = content[0].get("text", "") |
| 74 | elif isinstance(content, str): |
| 75 | text_content = content |
| 76 | else: |
| 77 | continue |
| 78 | |
| 79 | returncode_match = re.search(r"<returncode>(\d+)</returncode>", text_content) |
| 80 | |
| 81 | if i == 0 or not returncode_match: |
| 82 | continue |
| 83 | returncode = int(returncode_match.group(1)) |
| 84 | |
| 85 | # Extract bash command from code block |
| 86 | prev_message = self.messages[i - 1] |
| 87 | if prev_message["role"] != "assistant": |
| 88 | continue |
| 89 | prev_content = prev_message.get("content", "") |
| 90 | bash_match = re.search(r"```(bash|sh)\n(.*?)\n```", prev_content, re.DOTALL) |
| 91 | if not bash_match: |
| 92 | continue |
| 93 | command = bash_match.group(2).strip() |
| 94 | cmd_type = command.split()[0] if command else "unknown" |
| 95 | |
| 96 | if returncode != 0: |
| 97 | failed_commands += 1 |
| 98 | failed_command_types[cmd_type] = failed_command_types.get(cmd_type, 0) + 1 |
| 99 | |
| 100 | return {"failed_commands": failed_commands, "failed_command_types": failed_command_types} |
| 101 | |
| 102 | |
| 103 | def main(log_dir: str): |