Function findSimilarHelpers

src/codegraph/helper-extract.ts:94–179

(
  units: FunctionUnit[],
  opts: { minSim?: number; maxSim?: number; minTokens?: number } = {},
)

 * collapses a whole codebase of vaguely-alike boilerplate into one meaningless mega-cluster.
 */
export function findSimilarHelpers(
  units: FunctionUnit[],
  opts: { minSim?: number; maxSim?: number; minTokens?: number } = {},
): HelperCluster[] {
  const minSim = opts.minSim ?? 0.88;
  const maxSim = opts.maxSim ?? 1.0;
  const minTokens = opts.minTokens ?? 40;

  const base = units
    .map(u => {
      const norm = normalizeBody(u.body, u.name);
      const tokens = codeTokens(norm);
      return { u, norm, tf: termFreq(tokens), tok: tokens.length };
    })
    .filter(x => x.tok >= minTokens);

  // TF-IDF: weight each token by how DISTINCTIVE it is across the corpus. Ubiquitous structural
  // tokens (`.`, `(`, `const`, `=`) appear in nearly every function → idf≈0 → they stop dominating,
  // so only genuinely-shared logic (specific identifiers + call sequences) drives similarity. Without
  // this, every large function looks ~85% like every other and the whole thing over-clusters.
  const df = new Map<string, number>();
  for (const b of base) for (const t of b.tf.keys()) df.set(t, (df.get(t) ?? 0) + 1);
  const N = base.length;
  const idf = (t: string): number => Math.log((N + 1) / ((df.get(t) ?? 0) + 1)); // smoothed; token in all docs → ~0
  const weight = (tf: Map<string, number>): Map<string, number> => {
    const w = new Map<string, number>();
    for (const [t, f] of tf) { const iw = idf(t); if (iw > 0) w.set(t, f * iw); }
    return w;
  };

  const prepared = base
    // Weighted vector; fall back to raw tf if weighting empties it (degenerate all-tokens-universal case).
    .map(x => { const w = weight(x.tf); return { ...x, tf: w.size > 0 ? w : x.tf }; })
    // Larger functions first, so the biggest is the seed a cluster forms around (stable + meaningful).
    .sort((a, b) => b.tok - a.tok);

  // Seed-based clustering: each function joins the seed it's MOST similar to (≥ minSim), else seeds a new cluster.
  const seeded: { seed: number; members: number[] }[] = [];
  for (let i = 0; i < prepared.length; i++) {
    let best = -1, bestSim = -1;
    for (let c = 0; c < seeded.length; c++) {
      const s = cosineSim(prepared[i]!.tf, prepared[seeded[c]!.seed]!.tf);
      if (s >= minSim && s <= maxSim && s > bestSim) { bestSim = s; best = c; }
    }
    if (best >= 0) seeded[best]!.members.push(i);
    else seeded.push({ seed: i, members: [i] });
  }

  const clusters: HelperCluster[] = [];
  for (const { members: rawIdxs } of seeded) {
    // Drop members NESTED inside another member (same file, contained line range): a inner helper
    // shares most of its parent's tokens, so parent+child cluster together — but "extract the child
    // from its parent" is not a duplicate-consolidation opportunity. Keep the outermost.
    const idxs = rawIdxs.filter(i => {
      const a = prepared[i]!.u;
      return !rawIdxs.some(j => {
        if (j === i) return false;
        const b = prepared[j]!.u;