Function toEvalSummaries

scripts/discord-sample.ts:267–337

(exportData: EvaluationRunExport[])

}

function toEvalSummaries(exportData: EvaluationRunExport[]): EvalSummary[] {
  const evalMap = new Map<string, ModelSummary[]>();
  const labelMap = new Map<string, string>();

  exportData.forEach((run, index) => {
    assert(
      run !== null && typeof run === "object",
      `Invalid evaluation entry at index ${index}`,
    );

    const repo = run.evaluation?.repo;
    assert(
      typeof repo === "string" && repo.length > 0,
      `Missing evaluation repo for entry at index ${index}`,
    );

    const identifier = run.evaluation?.identifier ?? repo;
    assert(
      typeof identifier === "string" && identifier.length > 0,
      `Missing evaluation identifier for "${repo}" (index ${index})`,
    );

    assert(
      Array.isArray(run.scores),
      `Missing scores array for evaluation "${repo}" (index ${index})`,
    );

    assert(
      typeof run.jobUrl === "string" && run.jobUrl.length > 0,
      `Missing job URL for evaluation "${repo}" model "${run.model}"`,
    );

    const modelIds = Array.isArray(run.model) ? run.model : [run.model];
    const modelRows = run.scores.map((score) => ({
      name: score.assignment.name,
      weight: score.assignment.weight,
      normalizedWeight: score.normalizedWeight,
      average: score.averageScore,
      variance: score.variance,
    }));

    const summaries = evalMap.get(identifier) ?? [];

    modelIds.forEach((modelId) => {
      const agentName = (run.agent ?? "").trim();
      const prefixedId = `${agentName}:${modelId}`;

      summaries.push({
        id: prefixedId,
        rawModelId: modelId,
        agentId: agentName.length > 0 ? agentName : run.agent,
        final: run.finalScore,
        jobUrl: run.jobUrl,
        rows: modelRows,
        episodes: modelId.includes("claude") ? claudeEpisodes : gptEpisodes,
      });
    });