* Process documentation files and optionally save embeddings to database
(options: ProcessingOptions = {})
| 33 | * Process documentation files and optionally save embeddings to database |
| 34 | */ |
| 35 | async function processDocs(options: ProcessingOptions = {}) { |
| 36 | const config = { |
| 37 | docsPath: options.docsPath || path.join(process.cwd(), '../../apps/docs/content/docs/en'), |
| 38 | baseUrl: options.baseUrl || (isDev ? 'http://localhost:4000' : 'https://docs.sim.ai'), |
| 39 | chunkSize: options.chunkSize || 1024, |
| 40 | minCharactersPerChunk: options.minCharactersPerChunk || 100, |
| 41 | chunkOverlap: options.chunkOverlap || 200, |
| 42 | clearExisting: options.clearExisting ?? false, |
| 43 | dryRun: options.dryRun ?? false, |
| 44 | verbose: options.verbose ?? false, |
| 45 | } |
| 46 | |
| 47 | let processedChunks = 0 |
| 48 | let failedChunks = 0 |
| 49 | |
| 50 | try { |
| 51 | logger.info('🚀 Starting docs processing with config:', { |
| 52 | docsPath: config.docsPath, |
| 53 | baseUrl: config.baseUrl, |
| 54 | chunkSize: config.chunkSize, |
| 55 | clearExisting: config.clearExisting, |
| 56 | dryRun: config.dryRun, |
| 57 | }) |
| 58 | |
| 59 | // Initialize the chunker |
| 60 | const chunker = new DocsChunker({ |
| 61 | chunkSize: config.chunkSize, |
| 62 | minCharactersPerChunk: config.minCharactersPerChunk, |
| 63 | chunkOverlap: config.chunkOverlap, |
| 64 | baseUrl: config.baseUrl, |
| 65 | }) |
| 66 | |
| 67 | // Process all .mdx files |
| 68 | logger.info(`📚 Processing docs from: ${config.docsPath}`) |
| 69 | const chunks = await chunker.chunkAllDocs(config.docsPath) |
| 70 | |
| 71 | if (chunks.length === 0) { |
| 72 | logger.warn('⚠️ No chunks generated from docs') |
| 73 | return { success: false, processedChunks: 0, failedChunks: 0 } |
| 74 | } |
| 75 | |
| 76 | logger.info(`📊 Generated ${chunks.length} chunks with embeddings`) |
| 77 | |
| 78 | // Group chunks by document for summary |
| 79 | const chunksByDoc = chunks.reduce<Record<string, DocChunk[]>>((acc, chunk) => { |
| 80 | if (!acc[chunk.sourceDocument]) { |
| 81 | acc[chunk.sourceDocument] = [] |
| 82 | } |
| 83 | acc[chunk.sourceDocument].push(chunk) |
| 84 | return acc |
| 85 | }, {}) |
| 86 | |
| 87 | // Display summary |
| 88 | logger.info(`\n=== DOCUMENT SUMMARY ===`) |
| 89 | for (const [doc, docChunks] of Object.entries(chunksByDoc)) { |
| 90 | logger.info(`${doc}: ${docChunks.length} chunks`) |
| 91 | } |
| 92 |