Function benchmarkCrossSession

benchmark/scale-eval.ts:181–259

()

}

async function benchmarkCrossSession(): Promise<CrossSessionResult[]> {
  const { observations } = generateDataset();
  const results: CrossSessionResult[] = [];

  const bm25 = new SearchIndex();
  const kv = mockKV();
  const vector = new VectorIndex();
  const dims = 384;

  for (const obs of observations) {
    bm25.add(obs);
    const text = [obs.title, obs.narrative, ...obs.concepts].join(" ");
    vector.add(obs.id, obs.sessionId, deterministicEmbedding(text, dims));
    await kv.set(`mem:obs:${obs.sessionId}`, obs.id, obs);
  }

  const mockEmbed: any = {
    name: "deterministic", dimensions: dims,
    embed: async (t: string) => deterministicEmbedding(t, dims),
    embedBatch: async (ts: string[]) => ts.map(t => deterministicEmbedding(t, dims)),
  };
  const hybrid = new HybridSearch(bm25, vector, mockEmbed, kv as never, 0.4, 0.6, 0);

  const crossQueries: Array<{
    query: string;
    targetConcepts: string[];
    targetSessionRange: [number, number];
    currentSession: number;
  }> = [
    { query: "How did we set up OAuth providers?", targetConcepts: ["oauth", "nextauth"], targetSessionRange: [5, 9], currentSession: 29 },
    { query: "What was the N+1 query fix?", targetConcepts: ["n+1", "eager-loading"], targetSessionRange: [10, 14], currentSession: 28 },
    { query: "PostgreSQL full-text search setup", targetConcepts: ["full-text-search", "tsvector"], targetSessionRange: [10, 14], currentSession: 27 },
    { query: "bcrypt password hashing configuration", targetConcepts: ["bcrypt", "password-hashing"], targetSessionRange: [5, 9], currentSession: 25 },
    { query: "Vitest unit testing setup", targetConcepts: ["vitest", "unit-testing"], targetSessionRange: [20, 24], currentSession: 29 },
    { query: "webhook retry exponential backoff", targetConcepts: ["webhooks", "exponential-backoff"], targetSessionRange: [15, 19], currentSession: 29 },
    { query: "ESLint flat config migration", targetConcepts: ["eslint", "linting"], targetSessionRange: [0, 4], currentSession: 29 },
    { query: "Kubernetes HPA autoscaling configuration", targetConcepts: ["hpa", "autoscaling", "kubernetes"], targetSessionRange: [25, 29], currentSession: 29 },
    { query: "Prisma database seed script", targetConcepts: ["seeding", "faker", "prisma"], targetSessionRange: [10, 14], currentSession: 26 },
    { query: "API cursor-based pagination", targetConcepts: ["cursor-based", "pagination"], targetSessionRange: [15, 19], currentSession: 29 },
    { query: "CSRF protection double-submit cookie", targetConcepts: ["csrf", "cookies"], targetSessionRange: [5, 9], currentSession: 29 },
    { query: "blue-green deployment rollback", targetConcepts: ["blue-green", "rollback", "zero-downtime"], targetSessionRange: [25, 29], currentSession: 29 },
  ];

  for (const cq of crossQueries) {
    const targetObs = observations.filter(o =>
      o.concepts.some(c => cq.targetConcepts.includes(c))
    );
    const targetIds = new Set(targetObs.map(o => o.id));

    const start = performance.now();
    const bm25Results = bm25.search(cq.query, 20);
    const hybridResults = await hybrid.search(cq.query, 20);
    const latency = performance.now() - start;

    const bm25Rank = bm25Results.findIndex(r => targetIds.has(r.obsId));
    const hybridRank = hybridResults.findIndex(r => targetIds.has(r.observation.id));

    const builtinLines = 200;