Function addOverlap

apps/sim/lib/chunkers/utils.ts:23–52

(chunks: string[], overlapChars: number)

}

export function addOverlap(chunks: string[], overlapChars: number): string[] {
  if (overlapChars <= 0 || chunks.length <= 1) {
    return chunks
  }

  const result: string[] = []

  for (let i = 0; i < chunks.length; i++) {
    let chunk = chunks[i]

    if (i > 0) {
      const prevChunk = chunks[i - 1]
      const overlapLength = Math.min(overlapChars, prevChunk.length)
      const overlapText = prevChunk.slice(-overlapLength)

      const wordBoundaryMatch = overlapText.match(/^\s*\S/)
      const cleanOverlap = wordBoundaryMatch
        ? overlapText.slice(overlapText.indexOf(wordBoundaryMatch[0].trim()))
        : overlapText

      if (cleanOverlap.trim()) {
        chunk = `${cleanOverlap.trim()} ${chunk}`
      }
    }

    result.push(chunk)
  }

  return result
}

/**
 * When stepChars is provided (< chunkSizeChars), produces overlapping chunks