Function extractKeywords

src/api/utils/vectorStore.ts:136–187

* Extract keywords and their importance from text

(text: string)

 * Extract keywords and their importance from text
 */
function extractKeywords(text: string): Array<{ term: string; score: number }> {
  const results: Array<{ term: string; score: number }> = [];
  const words = text
    .toLowerCase()
    .split(/\W+/)
    .filter((w) => w.length > 3);

  // Count word frequencies
  const wordCounts: { [key: string]: number } = {};
  for (const word of words) {
    wordCounts[word] = (wordCounts[word] || 0) + 1;
  }

  // Find interesting terms (high frequency or within heading patterns)
  const headings = text.match(/#{1,6}\s+([^\n]+)/g) || [];
  const headingTerms = new Set<string>();

  // Extract terms from headings with higher weight
  for (const heading of headings) {
    const cleanHeading = heading.replace(/^#+\s+/, "").toLowerCase();
    const terms = cleanHeading.split(/\W+/).filter((w) => w.length > 3);
    terms.forEach((t) => headingTerms.add(t));
  }

  // Calculate term scores based on frequency and position
  const totalWords = words.length;

  for (const word in wordCounts) {
    // Skip common words or very rare words
    if (commonWords.has(word) || wordCounts[word] < 2) continue;

    // Calculate score based on frequency
    let score = wordCounts[word] / totalWords;

    // Boost score for terms in headings
    if (headingTerms.has(word)) {
      score *= 3;
    }

    // Boost for terms in the first paragraph (likely important)
    const firstPara = text.split("\n\n")[0].toLowerCase();
    if (firstPara.includes(word)) {
      score *= 1.5;
    }

    results.push({ term: word, score });
  }

  // Sort by score and take top 20
  results.sort((a, b) => b.score - a.score);
  return results.slice(0, 20);
}

/**
 * Normalize a vector to unit length