(args: {
config: RunnerConfig;
question: LongMemEvalQuestion;
questionState: QuestionRunState;
state: RunnerStateFile;
resultsMap: Map<string, QuestionResultRecord>;
})
| 658 | } |
| 659 | |
| 660 | async function judgeQuestion(args: { |
| 661 | config: RunnerConfig; |
| 662 | question: LongMemEvalQuestion; |
| 663 | questionState: QuestionRunState; |
| 664 | state: RunnerStateFile; |
| 665 | resultsMap: Map<string, QuestionResultRecord>; |
| 666 | }): Promise<QuestionResultRecord> { |
| 667 | const { config, question, questionState, state, resultsMap } = args; |
| 668 | const existing = resultsMap.get(question.question_id); |
| 669 | if (existing) { |
| 670 | return existing; |
| 671 | } |
| 672 | |
| 673 | questionState.status = "judging"; |
| 674 | questionState.updatedAt = nowIso(); |
| 675 | state.inProgress[question.question_id] = questionState; |
| 676 | await persistState(config, state); |
| 677 | |
| 678 | const hypothesis = questionState.finalQuestion.answerText ?? ""; |
| 679 | const judge = await judgeAnswer({ |
| 680 | config: config.judge, |
| 681 | question, |
| 682 | hypothesis, |
| 683 | }); |
| 684 | |
| 685 | addJudgeUsage(questionState, judge.usage, judge.estimatedCostUsd); |
| 686 | questionState.judge = { |
| 687 | model: judge.model, |
| 688 | prompt: judge.prompt, |
| 689 | rawResponse: judge.rawResponse, |
| 690 | label: judge.label, |
| 691 | }; |
| 692 | |
| 693 | const totalUsage = addTokenUsage(questionState.openCodeUsage, questionState.judgeUsage); |
| 694 | const result: QuestionResultRecord = { |
| 695 | question_id: question.question_id, |
| 696 | question_type: question.question_type, |
| 697 | dataset_index: questionState.datasetIndex, |
| 698 | question: question.question, |
| 699 | correct_answer: question.answer, |
| 700 | question_date: question.question_date, |
| 701 | hypothesis, |
| 702 | judgment: judge.label ? "yes" : "no", |
| 703 | score: judge.label ? 1 : 0, |
| 704 | autoeval_label: { |
| 705 | model: judge.model, |
| 706 | label: judge.label, |
| 707 | }, |
| 708 | answer_session_ids: question.answer_session_ids, |
| 709 | haystack_session_ids: question.haystack_session_ids, |
| 710 | haystack_open_code_session_ids: questionState.haystackSessions |
| 711 | .map((entry) => entry.openCodeSessionId) |
| 712 | .filter((value): value is string => typeof value === "string"), |
| 713 | final_open_code_session_id: questionState.finalQuestion.openCodeSessionId, |
| 714 | token_usage: { |
| 715 | openCode: questionState.openCodeUsage, |
| 716 | judge: questionState.judgeUsage, |
| 717 | total: totalUsage, |
no test coverage detected