| 40 | """ |
| 41 | |
| 42 | def predict(self, row: dict[str, Any]) -> TierVote: |
| 43 | messages = row.get("messages", []) |
| 44 | msg_count = len(messages) |
| 45 | has_tools = _has_tool_calls(messages) |
| 46 | tool_msg_count = _count_tool_related_messages(messages) |
| 47 | |
| 48 | # ─── Production-only heuristics ─── |
| 49 | # Uses ONLY: msg_count, has_tools, tool_msg_count |
| 50 | # NO benchmark, scenario, step_index, or total_steps |
| 51 | |
| 52 | # Strongest signal: tool usage (16% low vs 98% high) |
| 53 | if not has_tools: |
| 54 | if msg_count <= 3: |
| 55 | return TierVote(tier_id=0, confidence=0.75) |
| 56 | if msg_count <= 6: |
| 57 | return TierVote(tier_id=0, confidence=0.65) |
| 58 | return TierVote(tier_id=0, confidence=0.55) |
| 59 | |
| 60 | # Has tools — differentiate by depth. |
| 61 | # In multi-step agent workflows, tool messages accumulate from prior |
| 62 | # steps but the CURRENT step may be simple. The global "has_tools → |
| 63 | # tier 3" prior breaks here — per-step difficulty is not correlated |
| 64 | # with tool-call count. Dampen hard so the embedding signal (which |
| 65 | # reads the actual last-user content) can dominate. |
| 66 | multi_step_workflow = msg_count > 6 and tool_msg_count > 4 |
| 67 | conf_cap = 0.30 if multi_step_workflow else 1.0 |
| 68 | |
| 69 | if tool_msg_count >= 8 and msg_count > 10: |
| 70 | return TierVote(tier_id=3, confidence=min(0.75, conf_cap)) |
| 71 | |
| 72 | if tool_msg_count >= 4 and msg_count > 8: |
| 73 | return TierVote(tier_id=3, confidence=min(0.65, conf_cap)) |
| 74 | |
| 75 | if tool_msg_count >= 4: |
| 76 | return TierVote(tier_id=2, confidence=min(0.55, conf_cap)) |
| 77 | |
| 78 | if msg_count > 6: |
| 79 | return TierVote(tier_id=2, confidence=min(0.50, conf_cap)) |
| 80 | |
| 81 | # Light tools, short conversation → mid |
| 82 | return TierVote(tier_id=1, confidence=0.50) |