Function evaluateFixture

src/eval/harness.ts:126–140

({
  fixture,
  searcher,
  limit = 5,
  searchOptions
}: EvaluateFixtureParams)

}

export async function evaluateFixture({
  fixture,
  searcher,
  limit = 5,
  searchOptions
}: EvaluateFixtureParams): Promise<EvalSummary> {
  const results: EvalResult[] = [];

  for (const query of fixture.queries) {
    const searchResults = await searcher.search(query.query, limit, undefined, searchOptions);
    results.push(evaluateQuery(query, searchResults));
  }

  return summarizeEvaluation(results);
}

export function summarizeEvaluation(results: EvalResult[], gate: EvalGate = 0.7): EvalSummary {
  const total = results.length;