Function deduplicateContinuation

src/handler.ts:604–664

(existing: string, continuation: string)

 * 算法：从续写内容的头部取不同长度的前缀，检查是否出现在原内容的尾部
 */
export function deduplicateContinuation(existing: string, continuation: string): string {
    if (!continuation || !existing) return continuation;

    // 对比窗口：取原内容尾部和续写头部的最大重叠检测范围
    const maxOverlap = Math.min(500, existing.length, continuation.length);
    if (maxOverlap < 10) return continuation; // 太短不值得去重

    const tail = existing.slice(-maxOverlap);

    // 从长到短搜索重叠：找最长的匹配
    let bestOverlap = 0;
    for (let len = maxOverlap; len >= 10; len--) {
        const prefix = continuation.substring(0, len);
        // 检查 prefix 是否出现在 tail 的末尾
        if (tail.endsWith(prefix)) {
            bestOverlap = len;
            break;
        }
    }

    // 如果没找到尾部完全匹配的重叠，尝试行级别的去重
    // 场景：模型从某一行的开头重新开始，但截断点可能在行中间
    if (bestOverlap === 0) {
        const continuationLines = continuation.split('\n');
        const tailLines = tail.split('\n');
        
        // 从续写的第一行开始，在原内容尾部的行中寻找匹配
        if (continuationLines.length > 0 && tailLines.length > 0) {
            const firstContLine = continuationLines[0].trim();
            if (firstContLine.length >= 10) {
                // 检查续写的前几行是否在原内容尾部出现过
                for (let i = tailLines.length - 1; i >= 0; i--) {
                    if (tailLines[i].trim() === firstContLine) {
                        // 从这一行开始往后对比连续匹配的行数
                        let matchedLines = 1;
                        for (let k = 1; k < continuationLines.length && i + k < tailLines.length; k++) {
                            if (continuationLines[k].trim() === tailLines[i + k].trim()) {
                                matchedLines++;
                            } else {
                                break;
                            }
                        }
                        if (matchedLines >= 2) {
                            // 移除续写中匹配的行
                            const deduped = continuationLines.slice(matchedLines).join('\n');
                            // 行级去重记录到详细日志
                            return deduped;
                        }
                        break;
                    }
                }
            }
        }
    }

    if (bestOverlap > 0) {
        return continuation.substring(bestOverlap);
    }