Function buildSummary

packages/plugin/scripts/longmemeval/runner.ts:843–884

(results: QuestionResultRecord[], selectedCount: number, config: RunnerConfig)

}

function buildSummary(results: QuestionResultRecord[], selectedCount: number, config: RunnerConfig): RunSummary {
    const rows: RunSummaryRow[] = LONGMEMEVAL_QUESTION_TYPES.map((category) => {
        const items = results.filter((result) => result.question_type === category);
        const correct = items.reduce((sum, item) => sum + item.score, 0);
        return {
            category,
            correct,
            total: items.length,
            accuracy: items.length === 0 ? 0 : correct / items.length,
        };
    });

    const overallCorrect = results.reduce((sum, item) => sum + item.score, 0);
    rows.push({
        category: "overall",
        correct: overallCorrect,
        total: results.length,
        accuracy: results.length === 0 ? 0 : overallCorrect / results.length,
    });

    const tokens = results.reduce(
        (acc, result) => addTokenUsage(acc, result.token_usage.total),
        createEmptyTokenUsageStats(),
    );
    const totalActualUsd = results.reduce((sum, result) => sum + result.costs.totalActualUsd, 0);
    const totalEstimatedUsd = results.reduce((sum, result) => sum + result.costs.totalEstimatedUsd, 0);

    return {
        generatedAt: nowIso(),
        datasetPath: config.datasetPath,
        selectionSignature: config.selectionSignature,
        completedQuestions: results.length,
        selectedQuestions: selectedCount,
        pendingQuestions: Math.max(selectedCount - results.length, 0),
        rows,
        costs: {
            totalActualUsd,
            totalEstimatedUsd,
        },
        tokens,
    };
}

function printSummary(summary: RunSummary): void {
    console.log("\nLongMemEval summary");