Method enforceSizeLimit

apps/sim/lib/chunkers/docs-chunker.ts:354–386

(chunks: string[])

  }

  private enforceSizeLimit(chunks: string[]): string[] {
    const finalChunks: string[] = []

    for (const chunk of chunks) {
      const tokens = estimateTokens(chunk)

      if (tokens <= this.chunkSize) {
        finalChunks.push(chunk)
      } else {
        const lines = chunk.split('\n')
        let currentChunk = ''

        for (const line of lines) {
          const testChunk = currentChunk ? `${currentChunk}\n${line}` : line

          if (estimateTokens(testChunk) <= this.chunkSize) {
            currentChunk = testChunk
          } else {
            if (currentChunk.trim()) {
              finalChunks.push(currentChunk.trim())
            }
            currentChunk = line
          }
        }

        if (currentChunk.trim()) {
          finalChunks.push(currentChunk.trim())
        }
      }
    }

    return finalChunks.filter((chunk) => chunk.trim().length > 100)
  }
}