Function splitAtWordBoundaries

apps/sim/lib/chunkers/utils.ts:58–95

(
  text: string,
  chunkSizeChars: number,
  stepChars?: number
)

 * using a sliding window where chunks stay within the size limit.
 */
export function splitAtWordBoundaries(
  text: string,
  chunkSizeChars: number,
  stepChars?: number
): string[] {
  const parts: string[] = []
  let pos = 0

  while (pos < text.length) {
    let end = Math.min(pos + chunkSizeChars, text.length)

    if (end < text.length) {
      const lastSpace = text.lastIndexOf(' ', end)
      if (lastSpace > pos) {
        end = lastSpace
      }
    }

    const part = text.slice(pos, end).trim()
    if (part) {
      parts.push(part)
    }

    if (stepChars !== undefined) {
      // Sliding window: advance by step for predictable overlap
      const nextPos = pos + Math.max(1, stepChars)
      if (nextPos >= text.length) break
      pos = nextPos
    } else {
      // Non-overlapping: advance from end of extracted content
      if (end >= text.length) break
      pos = end
    }
    while (pos < text.length && text[pos] === ' ') pos++
  }

  return parts
}

export function buildChunks(texts: string[], overlapTokens: number): Chunk[] {
  let previousEndIndex = 0