Function main

packages/plugin/scripts/embedding-baseline-diff.ts:95–249

()

}

function main() {
    const [oldPath, newPath] = process.argv.slice(2);
    if (!oldPath || !newPath) {
        console.error(
            "Usage: bun packages/plugin/scripts/embedding-baseline-diff.ts <old.json> <new.json>",
        );
        process.exit(1);
    }

    const oldSnap = loadSnapshot(oldPath);
    const newSnap = loadSnapshot(newPath);

    console.log("");
    console.log("─── Snapshot meta ─────────────────────────────────────────");
    console.log(`OLD: ${oldSnap.modelId} (dim=${oldSnap.embeddingDim}) ${oldSnap.timestamp}`);
    console.log(`NEW: ${newSnap.modelId} (dim=${newSnap.embeddingDim}) ${newSnap.timestamp}`);
    console.log(
        `memories considered: old=${oldSnap.stats.memoriesConsidered} new=${newSnap.stats.memoriesConsidered}  topK=${oldSnap.stats.topK}`,
    );
    console.log(
        `avg query latency: old=${oldSnap.stats.queryEmbedLatencyMs.avg}ms new=${newSnap.stats.queryEmbedLatencyMs.avg}ms`,
    );

    // Build a map for quick lookup.
    const newById = new Map(newSnap.queries.map((q) => [q.id, q]));

    // Per-query metrics.
    interface Row {
        id: string;
        group: string;
        text: string;
        oldTop1: number;
        newTop1: number;
        oldTop10: number;
        newTop10: number;
        top1Changed: boolean;
        tau: number | null;
        overlapCount: number;
    }
    const rows: Row[] = [];

    for (const oldQ of oldSnap.queries) {
        const newQ = newById.get(oldQ.id);
        if (!newQ) continue;
        const oldIds = oldQ.topK.map((t) => t.memoryId);
        const newIds = newQ.topK.map((t) => t.memoryId);
        const newSet = new Set(newIds);
        const overlap = new Set(oldIds.filter((id) => newSet.has(id)));
        rows.push({
            id: oldQ.id,
            group: oldQ.group,
            text: oldQ.text,
            oldTop1: oldQ.topK[0]?.score ?? 0,
            newTop1: newQ.topK[0]?.score ?? 0,
            oldTop10: oldQ.topK[oldQ.topK.length - 1]?.score ?? 0,
            newTop10: newQ.topK[newQ.topK.length - 1]?.score ?? 0,
            top1Changed: oldIds[0] !== newIds[0],
            tau: kendallTau(oldIds, newIds),