Method chunk

apps/sim/lib/chunkers/text-chunker.ts:93–107

(text: string)

  }

  async chunk(text: string): Promise<Chunk[]> {
    if (!text?.trim()) {
      return []
    }

    const cleaned = cleanText(text)
    let chunks = this.splitRecursively(cleaned)

    if (this.chunkOverlap > 0) {
      const overlapChars = tokensToChars(this.chunkOverlap)
      chunks = addOverlap(chunks, overlapChars)
    }

    return buildChunks(chunks, this.chunkOverlap)
  }
}