Function dedupHistory

src/agent/dedup.ts:63–116

(messages: Message[], opts: DedupOptions = {})

 * Pure: returns a new array; the input is unchanged.
 */
export function dedupHistory(messages: Message[], opts: DedupOptions = {}): DedupResult {
  const minBytes = opts.minBytes ?? 200;
  const keepRecent = opts.keepRecent ?? 4;
  const safeTools = opts.extraSafeTools
    ? new Set([...SAFE_FOR_DEDUP, ...opts.extraSafeTools])
    : SAFE_FOR_DEDUP;

  // First pass: index every tool result by (toolName + contentHash) → first turn index.
  // We need toolName, which is stored on the tool message as `name` (we set this in the
  // agent loop when persisting). Without it, dedup is skipped for that message.
  type Indexed = { index: number; toolName: string; hash: string };
  const firstSeen = new Map<string, Indexed>(); // key = toolName + ':' + hash

  // Locate tool-message indices, in order
  const toolIdxs: number[] = [];
  for (let i = 0; i < messages.length; i++) {
    if (messages[i]!.role === 'tool') toolIdxs.push(i);
  }
  // Mark which tool messages are within the "keep recent" tail
  const tailStart = Math.max(0, toolIdxs.length - keepRecent);

  const out = messages.slice(); // shallow copy; we'll replace specific entries
  let replaced = 0;
  let bytesSaved = 0;

  for (let pos = 0; pos < toolIdxs.length; pos++) {
    const i = toolIdxs[pos]!;
    const msg = out[i]!;
    const content = msg.content ?? '';
    const toolName = (msg as any).name as string | undefined;
    if (!toolName || !safeTools.has(toolName)) continue;
    if (content.length < minBytes) continue;
    if (pos >= tailStart) continue; // keep recent results full

    const hash = hashContent(content);
    const key = toolName + ':' + hash;
    const seen = firstSeen.get(key);
    if (seen) {
      // Build a pointer line that's also useful to the model — same toolName + hash
      // means the model knows nothing changed without us saying it changed/didn't.
      const pointer =
        `[DEDUP] Same content as turn earlier in this session ` +
        `(tool=${toolName}, sha=${hash.slice(0, 10)}, ${content.length}B suppressed). ` +
        `Reuse what you already saw; call the tool again if you suspect the file changed.`;
      out[i] = { ...msg, content: pointer };
      replaced += 1;
      bytesSaved += content.length - pointer.length;
    } else {
      firstSeen.set(key, { index: i, toolName, hash });
    }
  }

  return { messages: out, replaced, bytesSaved };
}

function hashContent(s: string): string {
  return crypto.createHash('sha256').update(s).digest('hex');