Function evaluateChecks

src/eval/score.ts:66–102

(check: EvalCheck, outcome: Outcome)

/** Evaluate a task's checks against an observed outcome. Pure. */
export function evaluateChecks(check: EvalCheck, outcome: Outcome): CheckResult {
  const reasons: string[] = [];
  const existing = new Set(outcome.existingFiles);

  if (outcome.error) {
    reasons.push(`run error: ${outcome.error}`);
  }

  for (const p of check.filesExist ?? []) {
    if (!existing.has(p)) reasons.push(`expected file missing: ${p}`);
  }

  for (const fc of check.fileChecks ?? []) {
    const content = outcome.fileContents[fc.path];
    if (content == null) {
      reasons.push(`file not found for content check: ${fc.path}`);
      continue;
    }
    if (fc.contains != null && !content.includes(fc.contains)) {
      reasons.push(`${fc.path} does not contain "${fc.contains}"`);
    }
    if (fc.matches != null) {
      let re: RegExp | null = null;
      try { re = new RegExp(fc.matches); } catch { reasons.push(`invalid regex for ${fc.path}: ${fc.matches}`); }
      if (re && !re.test(content)) reasons.push(`${fc.path} does not match /${fc.matches}/`);
    }
  }

  if (check.command) {
    const expectZero = check.command.expectExitZero ?? true;
    const code = outcome.commandExitCode;
    if (expectZero && code !== 0) reasons.push(`command "${check.command.command}" exited ${code ?? 'null'} (expected 0)`);
    if (!expectZero && code === 0) reasons.push(`command "${check.command.command}" exited 0 (expected non-zero)`);
  }

  return { passed: reasons.length === 0, reasons };
}

export interface TaskRunResult {
  id: string;