Function dedupeSelfRepeatedText

src/cli/modes/final-dedupe.ts:133–185

(text: string)

 * block, never on incidental repetition of a sentence or a heading.
 */
export function dedupeSelfRepeatedText(text: string): string {
  const raw = text ?? '';
  if (raw.length < 400) return raw; // too short to contain a full double-answer

  // Find candidate restart points: a later occurrence of the text's own opening.
  // Use the first ~60 non-space chars as a fingerprint of the answer's start.
  const norm = (s: string) => s.replace(/\s+/g, ' ').trim();
  const head = raw.slice(0, Math.min(raw.length, 4000));
  const fpLen = 60;
  const fingerprint = norm(head).slice(0, fpLen);
  if (fingerprint.length < fpLen) return raw;

  // Search for the fingerprint appearing AGAIN after the first ~quarter of text.
  const normFull = norm(raw);
  const secondIdxNorm = normFull.indexOf(fingerprint, Math.floor(normFull.length * 0.25));
  if (secondIdxNorm === -1) return raw;

  // Map the normalized split point back to a raw offset by walking raw and
  // counting normalized chars. Cheaper approach: split raw on a literal search of
  // the original (un-normalized) opening line, which is what actually repeats.
  const openingLine = raw.split('\n').find(l => l.trim().length >= 12)?.trim();
  if (!openingLine) return raw;
  const firstAt = raw.indexOf(openingLine);
  const secondAt = raw.indexOf(openingLine, firstAt + openingLine.length);
  if (secondAt === -1) return raw;

  const first = raw.slice(firstAt, secondAt).trim();
  const second = raw.slice(secondAt).trim();
  // Confirm it's a real restart. The two copies are often NOT byte-identical —
  // e.g. the first ends with "Would you like me to…?" and the second doesn't, or
  // one is truncated. So instead of requiring exact containment, measure how much
  // of a shared opening they have: split each into normalized words and count the
  // common leading run. A genuine re-emit shares a long prefix; two different
  // sections that merely reuse one opening line diverge almost immediately.
  const ns = norm(first), nl = norm(second);
  if (ns.length < 200 || nl.length < 200) return raw;
  const wa = ns.split(' ');
  const wb = nl.split(' ');
  let common = 0;
  const max = Math.min(wa.length, wb.length);
  while (common < max && wa[common] === wb[common]) common++;
  // Require the shared opening run to be at least 60% of the shorter copy's word
  // count — strong evidence the model restarted the same answer, tolerant of a
  // differing tail.
  const sharedRatio = common / Math.min(wa.length, wb.length);
  if (common >= 30 && sharedRatio >= 0.6) {
    // Keep the prefix before the first copy (if any) + the longer single copy.
    const preamble = raw.slice(0, firstAt).trim();
    const body = first.length >= second.length ? first : second;
    return preamble ? `${preamble}\n\n${body}` : body;
  }
  return raw;
}

// Re-export so the headless module can keep its single import surface.
export { StreamDisplayFilter };