Method chunk

apps/sim/lib/chunkers/recursive-chunker.ts:129–144

(content: string)

  }

  async chunk(content: string): Promise<Chunk[]> {
    if (!content?.trim()) {
      return []
    }

    const cleaned = cleanText(content)
    let chunks = this.splitRecursively(cleaned)

    if (this.chunkOverlap > 0) {
      const overlapChars = tokensToChars(this.chunkOverlap)
      chunks = addOverlap(chunks, overlapChars)
    }

    logger.info(`Chunked into ${chunks.length} recursive chunks`)
    return buildChunks(chunks, this.chunkOverlap)
  }
}