Tokenize text for the FTS inverted index. Extracts lowercase alphanumeric words of length >= 3, filtering out stop words (common English + programming terms) that match too many nodes to be useful for ranking.
(text: &str)
| 1008 | /// stop words (common English + programming terms) that match too many |
| 1009 | /// nodes to be useful for ranking. |
| 1010 | pub fn tokenize_for_fts(text: &str) -> Vec<String> { |
| 1011 | text.split(|c: char| !c.is_alphanumeric() && c != '_') |
| 1012 | .filter(|w| w.len() >= 3) |
| 1013 | .map(|w| w.to_lowercase()) |
| 1014 | .filter(|w| !FTS_STOP_WORDS.contains(&w.as_str())) |
| 1015 | .collect() |
| 1016 | } |
| 1017 | |
| 1018 | /// Encode an embedding key as "path\0chunk_idx". |
| 1019 | #[inline] |