Method splitContent

apps/sim/lib/chunkers/docs-chunker.ts:193–211

Returns both chunks and cleaned content so header extraction uses aligned positions.

(
    content: string
  )

  /** Returns both chunks and cleaned content so header extraction uses aligned positions. */
  private async splitContent(
    content: string
  ): Promise<{ chunks: string[]; cleanedContent: string }> {
    const cleanedContent = this.cleanContent(content)

    const tableBoundaries = this.detectTableBoundaries(cleanedContent)

    const chunks = await this.textChunker.chunk(cleanedContent)

    const processedChunks = this.mergeTableChunks(
      chunks.map((chunk) => chunk.text),
      tableBoundaries,
      cleanedContent
    )

    const finalChunks = this.enforceSizeLimit(processedChunks)

    return { chunks: finalChunks, cleanedContent }
  }

  private cleanContent(content: string): string {
    return content