Method chunk

apps/sim/lib/chunkers/regex-chunker.ts:110–158

(content: string)

  }

  async chunk(content: string): Promise<Chunk[]> {
    if (!content?.trim()) {
      return []
    }

    const cleaned = cleanText(content)

    if (!this.strictBoundaries && estimateTokens(cleaned) <= this.chunkSize) {
      logger.info('Content fits in single chunk')
      return buildChunks([cleaned], 0)
    }

    this.regex.lastIndex = 0
    const segments = cleaned.split(this.regex).filter((s) => s.trim().length > 0)

    if (segments.length <= 1) {
      if (this.strictBoundaries) {
        logger.info('Regex pattern produced no splits in strict mode, returning single chunk')
        return buildChunks([cleaned.trim()], 0)
      }
      logger.warn(
        'Regex pattern did not produce any splits, falling back to word-boundary splitting'
      )
      const chunkSizeChars = tokensToChars(this.chunkSize)
      let chunks = splitAtWordBoundaries(cleaned, chunkSizeChars)
      if (this.chunkOverlap > 0) {
        const overlapChars = tokensToChars(this.chunkOverlap)
        chunks = addOverlap(chunks, overlapChars)
      }
      return buildChunks(chunks, this.chunkOverlap)
    }

    if (this.strictBoundaries) {
      const chunks = this.expandOversizedSegments(segments)
      logger.info(`Chunked into ${chunks.length} strict-boundary regex chunks`)
      return buildChunks(chunks, 0)
    }

    const merged = this.mergeSegments(segments)

    let chunks = merged
    if (this.chunkOverlap > 0) {
      const overlapChars = tokensToChars(this.chunkOverlap)
      chunks = addOverlap(chunks, overlapChars)
    }

    logger.info(`Chunked into ${chunks.length} regex-based chunks`)
    return buildChunks(chunks, this.chunkOverlap)
  }

  /**
   * In strict-boundary mode each segment becomes its own chunk. Segments that