Method chunkMdxFile

apps/sim/lib/chunkers/docs-chunker.ts:64–112

(filePath: string, basePath: string)

  }

  async chunkMdxFile(filePath: string, basePath: string): Promise<DocChunk[]> {
    const content = await fs.readFile(filePath, 'utf-8')
    const relativePath = path.relative(basePath, filePath)

    const { data: frontmatter, content: markdownContent } = this.parseFrontmatter(content)

    const documentUrl = this.generateDocumentUrl(relativePath)

    const { chunks: textChunks, cleanedContent } = await this.splitContent(markdownContent)

    const headers = this.extractHeaders(cleanedContent)

    logger.info(`Generating embeddings for ${textChunks.length} chunks in ${relativePath}`)
    const embeddingModel = getConfiguredEmbeddingModel()
    const embeddings: number[][] =
      textChunks.length > 0 ? (await generateEmbeddings(textChunks, embeddingModel)).embeddings : []

    const chunks: DocChunk[] = []
    let currentPosition = 0

    for (let i = 0; i < textChunks.length; i++) {
      const chunkText = textChunks[i]
      const chunkStart = currentPosition
      const chunkEnd = currentPosition + chunkText.length

      const relevantHeader = this.findRelevantHeader(headers, chunkStart)

      const chunk: DocChunk = {
        text: chunkText,
        tokenCount: estimateTokens(chunkText),
        sourceDocument: relativePath,
        headerLink: relevantHeader ? `${documentUrl}#${relevantHeader.anchor}` : documentUrl,
        headerText: relevantHeader?.text || frontmatter.title || 'Document Root',
        headerLevel: relevantHeader?.level || 1,
        embedding: embeddings[i] || [],
        embeddingModel,
        metadata: {
          startIndex: chunkStart,
          endIndex: chunkEnd,
          title: frontmatter.title,
        },
      }

      chunks.push(chunk)
      currentPosition = chunkEnd
    }

    return chunks
  }

  private async findMdxFiles(dirPath: string): Promise<string[]> {
    const files: string[] = []