(question: LongMemEvalQuestion, response: string)
| 46 | } |
| 47 | |
| 48 | export function buildJudgePrompt(question: LongMemEvalQuestion, response: string): string { |
| 49 | if (question.question_id.endsWith("_abs")) { |
| 50 | return fillTemplate(ABSTENTION_TEMPLATE, question.question, question.answer, response); |
| 51 | } |
| 52 | |
| 53 | switch (question.question_type) { |
| 54 | case "temporal-reasoning": |
| 55 | return fillTemplate(TEMPORAL_TEMPLATE, question.question, question.answer, response); |
| 56 | case "knowledge-update": |
| 57 | return fillTemplate(KNOWLEDGE_UPDATE_TEMPLATE, question.question, question.answer, response); |
| 58 | case "single-session-preference": |
| 59 | return fillTemplate(PREFERENCE_TEMPLATE, question.question, question.answer, response); |
| 60 | case "single-session-user": |
| 61 | case "single-session-assistant": |
| 62 | case "multi-session": |
| 63 | return fillTemplate(BASE_TEMPLATE, question.question, question.answer, response); |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | function getApiKey(config: JudgeConfig): string { |
| 68 | const value = process.env[config.apiKeyEnvVar]; |
no test coverage detected