Function tokenize

packages/text-search/src/tokenizer.ts:89–122

(text: string, options: TokenizeOptions = {})

 * tokenize("running dogs", { stem: true }) // ["run", "dog"]
 */
export function tokenize(text: string, options: TokenizeOptions = {}): Token[] {
  const { stem: applyStem = true, removeStopwords = true, minLength = 1 } = options;

  // Convert to lowercase and extract words
  const words = text
    .toLowerCase()
    .replace(/[^a-z0-9\s]/g, " ")
    .split(/\s+/)
    .filter((w) => w.length >= minLength);

  const tokens: Token[] = [];

  for (let i = 0; i < words.length; i++) {
    const word = words[i]!;

    // Skip stopwords if configured
    if (removeStopwords && STOPWORDS.has(word)) {
      continue;
    }

    // Skip very short words
    if (word.length < minLength) {
      continue;
    }

    tokens.push({
      original: word,
      stemmed: applyStem ? stem(word) : word,
      position: i,
    });
  }

  return tokens;
}

/**
 * Extract just the stemmed terms from text.