Function tokenize_for_fts

atomic-core/src/pristine/tables.rs:796–802

Tokenize text for the FTS inverted index. Extracts lowercase alphanumeric words of length >= 3, filtering out stop words (common English + programming terms) that match too many nodes to be useful for ranking.

(text: &str)

/// stop words (common English + programming terms) that match too many
/// nodes to be useful for ranking.
pub fn tokenize_for_fts(text: &str) -> Vec<String> {
    text.split(|c: char| !c.is_alphanumeric() && c != '_')
        .filter(|w| w.len() >= 3)
        .map(|w| w.to_lowercase())
        .filter(|w| !FTS_STOP_WORDS.contains(&w.as_str()))
        .collect()
}

/// Encode an embedding key as "path\0chunk_idx".
#[inline]