Function processDocument

apps/sim/lib/knowledge/documents/document-processor.ts:170–264

(
  fileUrl: string,
  filename: string,
  mimeType: string,
  chunkSize = 1024,
  chunkOverlap = 200,
  minCharactersPerChunk = 100,
  userId?: string,
  workspaceId?: string | null,
  strategy?: ChunkingStrategy,
  strategyOptions?: StrategyOptions
)

}

export async function processDocument(
  fileUrl: string,
  filename: string,
  mimeType: string,
  chunkSize = 1024,
  chunkOverlap = 200,
  minCharactersPerChunk = 100,
  userId?: string,
  workspaceId?: string | null,
  strategy?: ChunkingStrategy,
  strategyOptions?: StrategyOptions
): Promise<{
  chunks: Chunk[]
  metadata: {
    filename: string
    fileSize: number
    mimeType: string
    chunkCount: number
    tokenCount: number
    characterCount: number
    processingMethod: 'file-parser' | 'mistral-ocr'
    cloudUrl?: string
  }
}> {
  logger.info(`Processing document: ${filename}`)

  try {
    const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId)
    const { content, processingMethod } = parseResult
    const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined

    let chunks: Chunk[]
    const metadata: FileParseMetadata = parseResult.metadata ?? {}

    if (strategy && strategy !== 'auto') {
      logger.info(`Using explicit chunking strategy: ${strategy}`)
      chunks = await applyStrategy(
        strategy,
        content,
        chunkSize,
        chunkOverlap,
        minCharactersPerChunk,
        strategyOptions
      )
    } else {
      const isJsonYaml =
        metadata.type === 'json' ||
        metadata.type === 'yaml' ||
        mimeType.includes('json') ||
        mimeType.includes('yaml')

      if (isJsonYaml && JsonYamlChunker.isStructuredData(content)) {
        logger.info('Using JSON/YAML chunker for structured data')
        chunks = await JsonYamlChunker.chunkJsonYaml(content, {
          chunkSize,
          minCharactersPerChunk,
        })
      } else if (StructuredDataChunker.isStructuredData(content, mimeType)) {