Method chunk

apps/sim/lib/chunkers/token-chunker.ts:26–53

(content: string)

  }

  async chunk(content: string): Promise<Chunk[]> {
    if (!content?.trim()) {
      return []
    }

    const cleaned = cleanText(content)

    if (estimateTokens(cleaned) <= this.chunkSize) {
      logger.info('Content fits in single chunk')
      return buildChunks([cleaned], 0)
    }

    const chunkSizeChars = tokensToChars(this.chunkSize)
    const overlapChars = tokensToChars(this.chunkOverlap)
    const stepChars = this.chunkOverlap > 0 ? chunkSizeChars - overlapChars : undefined

    const rawChunks = splitAtWordBoundaries(cleaned, chunkSizeChars, stepChars)

    const filtered =
      rawChunks.length > 1
        ? rawChunks.filter((c) => c.length >= this.minCharactersPerChunk)
        : rawChunks

    const chunks = filtered.length > 0 ? filtered : rawChunks

    logger.info(`Chunked into ${chunks.length} token-based chunks`)
    return buildChunks(chunks, this.chunkOverlap)
  }
}