Return a list of failure reason labels for a normalized result.
(result: NormalizedResult)
| 51 | |
| 52 | |
| 53 | def classify_failure(result: NormalizedResult) -> list[str]: |
| 54 | """Return a list of failure reason labels for a normalized result.""" |
| 55 | reasons: list[str] = [] |
| 56 | commands = [str(command) for command in result.commands] |
| 57 | history = [str(item) for item in result.history] |
| 58 | combined_text = "\n".join(commands + history) |
| 59 | first_command = commands[0] if commands else "" |
| 60 | |
| 61 | if not first_command or "xray ws --url" not in first_command: |
| 62 | reasons.append("wrong_command") |
| 63 | if len(commands) < 6: |
| 64 | reasons.append("give_up_early") |
| 65 | if "Invalid Format: Missing 'Action:' after 'Thought:" in combined_text: |
| 66 | reasons.append("tool_failure") |
| 67 | if "I cannot assist with that" in combined_text or "I can’t assist with that" in combined_text: |
| 68 | reasons.append("security_review") |
| 69 | if "Error code: 400" in combined_text: |
| 70 | reasons.append("context_limitation") |
| 71 | |
| 72 | if not reasons: |
| 73 | reasons.append("other") |
| 74 | return reasons |
no outgoing calls
no test coverage detected