(item)
| 88 | // 19. Process and vectorize the content |
| 89 | let vectorCount = 0; |
| 90 | const fetchAndProcess = async (item) => { |
| 91 | const htmlContent = await fetchPageContent(item.link); |
| 92 | if (htmlContent && htmlContent.length < 250) return null; |
| 93 | const splitText = await new RecursiveCharacterTextSplitter({ chunkSize: textChunkSize, chunkOverlap: textChunkOverlap }).splitText(htmlContent); |
| 94 | const vectorStore = await MemoryVectorStore.fromTexts(splitText, { link: item.link, title: item.title }, embeddings); |
| 95 | vectorCount++; |
| 96 | console.log(`9. Processed ${vectorCount} sources for ${item.link}`); |
| 97 | return await vectorStore.similaritySearch(message, numberOfSimilarityResults); |
| 98 | }; |
| 99 | // 20. Fetch and process sources |
| 100 | const sources = await searchEngineForSources(message, textChunkSize, textChunkOverlap); |
| 101 | const sourcesParsed = sources.map(group => |
nothing calls this directly
no test coverage detected