| 3414 | failed++; |
| 3415 | } |
| 3416 | /* An interrupted run whose partial output happens to look correct must not |
| 3417 | * be counted or flagged as drift. */ |
| 3418 | if (regrade_case_outcome(&integer, "STOPPED", "</think>Answer: 82", |
| 3419 | got, sizeof(got)) != REGRADE_NOT_GRADED) { |
| 3420 | fprintf(stderr, "ds4-eval: regrade self-test failed: STOPPED trace was graded\n"); |
| 3421 | failed++; |
| 3422 | } |
| 3423 | /* Legacy traces without a status line stay gradeable. */ |
| 3424 | if (regrade_case_outcome(&integer, "", "</think>Answer: 82", |
| 3425 | got, sizeof(got)) != REGRADE_PASSED) { |
| 3426 | fprintf(stderr, "ds4-eval: regrade self-test failed: legacy empty status not graded\n"); |
| 3427 | failed++; |
| 3428 | } |
| 3429 | return failed; |
| 3430 | } |
| 3431 | |
| 3432 | static int run_extractor_self_tests(void) { |
| 3433 | int failed = 0; |
| 3434 | |
| 3435 | failed += trace_copy_self_test_case(); |
| 3436 | failed += regrade_status_self_test_case(); |
| 3437 | |
| 3438 | const eval_case mc = { |
| 3439 | .source = "SuperGPQA", |
| 3440 | .choice[0] = "A", |
| 3441 | .choice[1] = "B", |
| 3442 | .choice[2] = "C", |
| 3443 | .choice[3] = "D", |
| 3444 | .choice[4] = "E", |
| 3445 | .choice[5] = "F", |
| 3446 | .choice[6] = "G", |
| 3447 | .choice[7] = "H", |
| 3448 | .choice[8] = "I", |
| 3449 | .choice[9] = "J", |
| 3450 | .answer = "F", |
| 3451 | }; |
| 3452 | failed += extractor_self_test_case( |
| 3453 | "multiple-choice prefers final answer marker", |
| 3454 | &mc, |
| 3455 | "</think>So answer is 0.716 H+/O2. That corresponds to option F.\n" |
| 3456 | "Thus final answer: F.</think>The visible explanation repeats the calculation.\n" |
| 3457 | "Answer: F", |
| 3458 | "F"); |
| 3459 | failed += extractor_self_test_case( |
| 3460 | "multiple-choice prefers Answer-colon over later prose", |
| 3461 | &mc, |
| 3462 | "</think>Answer: F\nThis answer is final; option H is a tempting distractor.", |
| 3463 | "F"); |
| 3464 | failed += extractor_self_test_case( |
| 3465 | "multiple-choice preserves loose-answer fallback", |
| 3466 | &mc, |
| 3467 | "</think>The answer is F. This answer is final; option H is tempting.", |
| 3468 | "F"); |
| 3469 | |
| 3470 | const eval_case integer = { |
| 3471 | .source = "AIME2025", |
| 3472 | .answer = "82", |
| 3473 | }; |
no test coverage detected