Method mergeTableChunks

apps/sim/lib/chunkers/docs-chunker.ts:302–352

(
    chunks: string[],
    tableBoundaries: { start: number; end: number }[],
    originalContent: string
  )

  }

  private mergeTableChunks(
    chunks: string[],
    tableBoundaries: { start: number; end: number }[],
    originalContent: string
  ): string[] {
    if (tableBoundaries.length === 0) {
      return chunks
    }

    const mergedChunks: string[] = []
    let currentPosition = 0

    for (const chunk of chunks) {
      const chunkStart = originalContent.indexOf(chunk, currentPosition)
      if (chunkStart === -1) {
        mergedChunks.push(chunk)
        continue
      }
      const chunkEnd = chunkStart + chunk.length

      const intersectsTable = tableBoundaries.some(
        (table) =>
          (chunkStart >= table.start && chunkStart <= table.end) ||
          (chunkEnd >= table.start && chunkEnd <= table.end) ||
          (chunkStart <= table.start && chunkEnd >= table.end)
      )

      if (intersectsTable) {
        const affectedTables = tableBoundaries.filter(
          (table) =>
            (chunkStart >= table.start && chunkStart <= table.end) ||
            (chunkEnd >= table.start && chunkEnd <= table.end) ||
            (chunkStart <= table.start && chunkEnd >= table.end)
        )

        const minStart = Math.min(chunkStart, ...affectedTables.map((t) => t.start))
        const maxEnd = Math.max(chunkEnd, ...affectedTables.map((t) => t.end))
        const completeChunk = originalContent.slice(minStart, maxEnd).trim()

        if (completeChunk && !mergedChunks.some((existing) => existing === completeChunk)) {
          mergedChunks.push(completeChunk)
        }
      } else {
        mergedChunks.push(chunk)
      }

      currentPosition = chunkEnd
    }

    return mergedChunks.filter((chunk) => chunk.length > 50)
  }

  private enforceSizeLimit(chunks: string[]): string[] {
    const finalChunks: string[] = []