| 3240 | * model output and must be reported as not-graded rather than counted, since |
| 3241 | * grading them inflates the totals and raises spurious "changed" drift. An |
| 3242 | * empty status keeps legacy traces (written before the status line) working. */ |
| 3243 | static regrade_outcome regrade_case_outcome(const eval_case *tc, |
| 3244 | const char *traced_status, |
| 3245 | const char *model_output, |
| 3246 | char *got, size_t gotlen) { |
| 3247 | bool gradeable = traced_status[0] == '\0' || |
| 3248 | !strcmp(traced_status, "PASSED") || |
| 3249 | !strcmp(traced_status, "FAILED"); |
| 3250 | if (!gradeable) { |
| 3251 | if (gotlen) got[0] = '\0'; |
| 3252 | return REGRADE_NOT_GRADED; |
| 3253 | } |
| 3254 | find_case_answer(tc, model_output, got, gotlen); |
| 3255 | return answer_matches(tc, got) ? REGRADE_PASSED : REGRADE_FAILED; |
| 3256 | } |
| 3257 | |
| 3258 | static int regrade_trace_file(const char *path) { |
| 3259 | size_t len = 0; |
| 3260 | char *text = read_text_file(path, &len); |
| 3261 | if (!text) return 2; |
| 3262 | |
| 3263 | const char *start = text; |
| 3264 | const char *end = text + len; |
| 3265 | int total = 0; |
| 3266 | int passed = 0; |
| 3267 | int failed = 0; |
| 3268 | int changed = 0; |
| 3269 | int unknown = 0; |
| 3270 | int parse_errors = 0; |
| 3271 | int not_graded = 0; |
| 3272 | |
| 3273 | while (true) { |
| 3274 | const char *case_start = trace_find_next_case(start, end); |
| 3275 | if (!case_start) break; |
| 3276 | const char *after_header = memchr(case_start, '\n', (size_t)(end - case_start)); |
| 3277 | after_header = after_header ? after_header + 1 : end; |
| 3278 | const char *case_end = trace_find_next_case(after_header, end); |
| 3279 | if (!case_end) case_end = end; |
| 3280 | start = case_end; |
| 3281 | total++; |
| 3282 | |
| 3283 | char source[64]; |
| 3284 | char id[128]; |
| 3285 | char traced_status[32]; |
| 3286 | char traced_pick[EVAL_ANSWER_MAX]; |
| 3287 | if (!trace_get_line_field(case_start, case_end, "source: ", source, sizeof(source)) || |
| 3288 | !trace_get_line_field(case_start, case_end, "id: ", id, sizeof(id))) { |
| 3289 | fprintf(stderr, "ds4-eval: trace case %d is missing source/id\n", total); |
| 3290 | parse_errors++; |
| 3291 | continue; |
| 3292 | } |
| 3293 | trace_get_line_field(case_start, case_end, "status: ", traced_status, sizeof(traced_status)); |
| 3294 | trace_get_line_field(case_start, case_end, "picked: ", traced_pick, sizeof(traced_pick)); |
| 3295 | |
| 3296 | const eval_case *tc = find_eval_case_by_source_id(source, id); |
| 3297 | if (!tc) { |
| 3298 | fprintf(stderr, "ds4-eval: trace case %d not found in embedded cases: %s/%s\n", |
| 3299 | total, source, id); |
no test coverage detected