| 87 | |
| 88 | @dataclass |
| 89 | class EvalResult: |
| 90 | prompts: list[EvalPrompt] = field(default_factory=list) |
| 91 | misses: list[EvalMiss] = field(default_factory=list) |
| 92 | coverage_misses: list[CoverageMiss] = field(default_factory=list) |
| 93 | # Trigger prompts where the coverage grader returned an unparseable |
| 94 | # verdict (neither SUPPORTED nor UNSUPPORTED). Tracked separately so |
| 95 | # grader-malfunction doesn't silently inflate ``coverage_misses`` and |
| 96 | # deflate ``coverage_rate``. |
| 97 | coverage_ambiguous: list[CoverageMiss] = field(default_factory=list) |
| 98 | # Prompts whose grader raised (typically ``RuntimeError`` from a |
| 99 | # ``MaxTurnsExceeded`` wrap or a malformed-response failure inside |
| 100 | # the SDK). Captured per-task via ``return_exceptions=True`` in |
| 101 | # ``run_eval`` so one failure doesn't discard the other ~29 |
| 102 | # gradings. Errors are excluded from rate denominators — we don't |
| 103 | # know what the verdict would have been. |
| 104 | trigger_errors: list[CoverageMiss] = field(default_factory=list) |
| 105 | coverage_errors: list[CoverageMiss] = field(default_factory=list) |
| 106 | |
| 107 | @property |
| 108 | def total(self) -> int: |
| 109 | return len(self.prompts) |
| 110 | |
| 111 | @property |
| 112 | def passed(self) -> int: |
| 113 | return self.trigger_scored - len(self.misses) |
| 114 | |
| 115 | @property |
| 116 | def trigger_scored(self) -> int: |
| 117 | """Trigger prompts the grader returned a verdict on (no error).""" |
| 118 | return self.total - len(self.trigger_errors) |
| 119 | |
| 120 | @property |
| 121 | def pass_rate(self) -> float: |
| 122 | scored = self.trigger_scored |
| 123 | return self.passed / scored if scored else 0.0 |
| 124 | |
| 125 | @property |
| 126 | def trigger_questions(self) -> int: |
| 127 | return sum(1 for p in self.prompts if p.expected == "trigger") |
| 128 | |
| 129 | @property |
| 130 | def coverage_passed(self) -> int: |
| 131 | # Ambiguous and errored outputs are excluded from both numerator |
| 132 | # and denominator — see ``coverage_rate``. |
| 133 | scored = self.trigger_questions - len(self.coverage_ambiguous) - len(self.coverage_errors) |
| 134 | return scored - len(self.coverage_misses) |
| 135 | |
| 136 | @property |
| 137 | def coverage_rate(self) -> float: |
| 138 | # Score only the trigger prompts the grader gave a clear verdict |
| 139 | # on. A garbled run that flips half the outputs to ambiguous or |
| 140 | # errors out should narrow the denominator, not pretend half the |
| 141 | # body is hollow. |
| 142 | scored = self.trigger_questions - len(self.coverage_ambiguous) - len(self.coverage_errors) |
| 143 | return self.coverage_passed / scored if scored else 0.0 |
| 144 | |
| 145 | |
| 146 | def _read_description(skill_dir: Path) -> str: |