Function tokenizeForSearch

src/context/hybrid-search.ts:33–49

(text: string)

 * query "user id" matches an identifier "getUserById" / "user_id". Lowercased.
 */
export function tokenizeForSearch(text: string): string[] {
  const out: string[] = [];
  // First split on non-alphanumeric, then break camelCase within each piece.
  const rawTokens = text.split(/[^A-Za-z0-9]+/).filter(Boolean);
  for (const raw of rawTokens) {
    out.push(raw.toLowerCase());
    // camelCase / PascalCase → sub-words (getUserById → get, user, by, id)
    const camelParts = raw.replace(/([a-z0-9])([A-Z])/g, '$1 $2').split(/\s+/).filter(Boolean);
    if (camelParts.length > 1) {
      for (const p of camelParts) {
        const lower = p.toLowerCase();
        if (lower !== raw.toLowerCase()) out.push(lower);
      }
    }
  }
  return out;
}

// ── BM25 ─────────────────────────────────────────────────────────────────────