Method chunkArray

apps/sim/lib/chunkers/json-yaml-chunker.ts:89–143

(arr: JsonArray, path: string[], depth: number)

  }

  private chunkArray(arr: JsonArray, path: string[], depth: number): Chunk[] {
    const chunks: Chunk[] = []
    let currentBatch: JsonValue[] = []
    let currentTokens = 0

    const contextHeader = path.length > 0 ? `// ${path.join('.')}\n` : ''

    for (let i = 0; i < arr.length; i++) {
      const item = arr[i]
      const itemStr = JSON.stringify(item, null, 2)
      const itemTokens = estimateTokens(itemStr)

      if (itemTokens > this.chunkSize) {
        if (currentBatch.length > 0) {
          chunks.push(
            this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
          )
          currentBatch = []
          currentTokens = 0
        }

        if (depth < MAX_DEPTH && typeof item === 'object' && item !== null) {
          chunks.push(...this.chunkStructuredData(item, [...path, `[${i}]`], depth + 1))
        } else {
          chunks.push({
            text: contextHeader + itemStr,
            tokenCount: itemTokens,
            metadata: { startIndex: i, endIndex: i },
          })
        }
      } else if (currentTokens + itemTokens > this.chunkSize && currentBatch.length > 0) {
        chunks.push(
          this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
        )
        currentBatch = [item]
        currentTokens = itemTokens
      } else {
        currentBatch.push(item)
        currentTokens += itemTokens
      }
    }

    if (currentBatch.length > 0) {
      chunks.push(
        this.buildBatchChunk(
          contextHeader,
          currentBatch,
          arr.length - currentBatch.length,
          arr.length - 1
        )
      )
    }

    return chunks
  }

  private chunkObject(obj: JsonObject, path: string[], depth: number): Chunk[] {
    const chunks: Chunk[] = []