Function judgeAnswer

packages/plugin/scripts/longmemeval/judge.ts:106–148

(args: {
    config: JudgeConfig;
    question: LongMemEvalQuestion;
    hypothesis: string;
    signal?: AbortSignal;
})

}

export async function judgeAnswer(args: {
    config: JudgeConfig;
    question: LongMemEvalQuestion;
    hypothesis: string;
    signal?: AbortSignal;
}): Promise<JudgeResult> {
    const prompt = buildJudgePrompt(args.question, args.hypothesis);
    const response = await fetch(`${args.config.apiBaseUrl.replace(/\/$/, "")}/chat/completions`, {
        method: "POST",
        headers: {
            "content-type": "application/json",
            authorization: `Bearer ${getApiKey(args.config)}`,
        },
        body: JSON.stringify({
            model: args.config.model,
            messages: [{ role: "user", content: prompt }],
            n: 1,
            temperature: args.config.temperature,
            max_tokens: args.config.maxTokens,
        }),
        signal: args.signal,
    });

    if (!response.ok) {
        const errorText = await response.text();
        throw new Error(`Judge request failed (${response.status}): ${errorText}`);
    }

    const payload = (await response.json()) as OpenAIChatCompletionResponse;
    const rawResponse = extractResponseText(payload);
    const label = rawResponse.toLowerCase().includes("yes");
    const usage = payload.usage ? extractUsage(payload) : createEmptyTokenUsageStats();
    const estimatedCostUsd = estimateCostFromPricing(usage, args.config.pricing);

    return {
        model: args.config.model,
        prompt,
        rawResponse,
        label,
        usage,
        estimatedCostUsd,
    };
}