Function evalDualStream

benchmark/quality-eval.ts:173–240

(
  observations: CompressedObservation[],
  queries: LabeledQuery[],
)

}

async function evalDualStream(
  observations: CompressedObservation[],
  queries: LabeledQuery[],
): Promise<SystemMetrics> {
  const kv = mockKV();
  const bm25 = new SearchIndex();
  const vector = new VectorIndex();
  const dims = 384;

  for (const obs of observations) {
    bm25.add(obs);
    const text = [obs.title, obs.narrative, ...obs.concepts, ...obs.facts].join(" ");
    vector.add(obs.id, obs.sessionId, deterministicEmbedding(text, dims));
    await kv.set(`mem:obs:${obs.sessionId}`, obs.id, obs);
  }

  const mockEmbed: any = {
    name: "deterministic",
    dimensions: dims,
    embed: async (text: string) => deterministicEmbedding(text, dims),
    embedBatch: async (texts: string[]) => texts.map(t => deterministicEmbedding(t, dims)),
  };

  const hybrid = new HybridSearch(bm25, vector, mockEmbed, kv as never, 0.4, 0.6, 0);
  const perQuery: QualityMetrics[] = [];

  for (const q of queries) {
    const relevant = new Set(q.relevantObsIds);
    const start = performance.now();
    const results = await hybrid.search(q.query, 20);
    const latency = performance.now() - start;

    const retrieved = results.map(r => r.observation.id);
    perQuery.push({
      query: q.query,
      category: q.category,
      recall_at_5: recall(retrieved, relevant, 5),
      recall_at_10: recall(retrieved, relevant, 10),
      recall_at_20: recall(retrieved, relevant, 20),
      precision_at_5: precision(retrieved, relevant, 5),
      precision_at_10: precision(retrieved, relevant, 10),
      ndcg_at_10: ndcg(retrieved, relevant, 10),
      mrr: mrr(retrieved, relevant),
      relevant_count: relevant.size,
      retrieved_count: results.length,
      latency_ms: latency,
    });
  }

  const avgResultTokens = perQuery.reduce((sum, q) => {
    return sum + q.retrieved_count;
  }, 0) / perQuery.length;
  const avgObsTokens2 = observations.slice(0, 50).reduce((s, o) => s + estimateTokens(JSON.stringify(o)), 0) / 50;

  return {
    system: "Dual-stream (BM25+Vector)",
    avg_recall_at_5: avg(perQuery.map(q => q.recall_at_5)),
    avg_recall_at_10: avg(perQuery.map(q => q.recall_at_10)),