Function benchmarkModel

implementations/browser/embedding-bench.mjs:123–215

(modelName, dims)

}

async function benchmarkModel(modelName, dims) {
  console.log(`\n${'='.repeat(60)}`);
  console.log(`MODEL: ${modelName} (${dims}d)`);
  console.log('='.repeat(60));

  const loadStart = performance.now();
  const embedder = await pipeline('feature-extraction', modelName, {
    quantized: true,
  });
  const loadTime = ((performance.now() - loadStart) / 1000).toFixed(1);
  console.log(`Loaded in ${loadTime}s`);

  // Embed all packages
  const pkgIds = Object.keys(PACKAGES);
  const pkgTexts = Object.values(PACKAGES);
  const pkgEmbeddings = {};

  for (let i = 0; i < pkgIds.length; i++) {
    const result = await embedder(pkgTexts[i], { pooling: 'mean', normalize: true });
    pkgEmbeddings[pkgIds[i]] = Array.from(result.data);
  }

  // Benchmark each conversation
  let correct = 0;
  let total = 0;
  const latencies = [];

  console.log('\n  Results:');

  for (const [convName, messages] of Object.entries(CONVERSATIONS)) {
    const convText = formatConversation(messages);

    // Measure latency (average of 5 runs)
    const times = [];
    let embedding;
    for (let i = 0; i < 5; i++) {
      const start = performance.now();
      const result = await embedder(convText, { pooling: 'mean', normalize: true });
      times.push(performance.now() - start);
      embedding = Array.from(result.data);
    }
    const avgMs = times.slice(1).reduce((a, b) => a + b, 0) / (times.length - 1);
    latencies.push(avgMs);

    // Score against all packages
    const scores = {};
    for (const [pkgId, pkgEmb] of Object.entries(pkgEmbeddings)) {
      scores[pkgId] = cosineSim(embedding, pkgEmb);
    }

    // Rank
    const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
    const top = ranked[0];
    const expected = EXPECTED[convName];

    if (expected) {
      total++;
      const isCorrect = top[0] === expected;