Classify a single cleaned token and push any symbols it yields.
(
clean: &str,
stop_words: &HashSet<&str>,
symbols: &mut Vec<String>,
seen: &mut HashSet<String>,
)
| 679 | |
| 680 | /// Classify a single cleaned token and push any symbols it yields. |
| 681 | fn classify_token( |
| 682 | clean: &str, |
| 683 | stop_words: &HashSet<&str>, |
| 684 | symbols: &mut Vec<String>, |
| 685 | seen: &mut HashSet<String>, |
| 686 | ) { |
| 687 | if clean.is_empty() { |
| 688 | return; |
| 689 | } |
| 690 | |
| 691 | if clean.contains("::") { |
| 692 | // Qualified path: extract last segment and full path |
| 693 | if let Some(last) = clean.rsplit("::").next() { |
| 694 | if !last.is_empty() |
| 695 | && !stop_words.contains(last.to_lowercase().as_str()) |
| 696 | && seen.insert(last.to_string()) |
| 697 | { |
| 698 | symbols.push(last.to_string()); |
| 699 | } |
| 700 | } |
| 701 | let full = clean.to_string(); |
| 702 | if seen.insert(full.clone()) { |
| 703 | symbols.push(full); |
| 704 | } |
| 705 | return; |
| 706 | } |
| 707 | |
| 708 | // snake_case or SCREAMING_SNAKE |
| 709 | if clean.contains('_') { |
| 710 | if !stop_words.contains(clean.to_lowercase().as_str()) && seen.insert(clean.to_string()) { |
| 711 | symbols.push(clean.to_string()); |
| 712 | } |
| 713 | // Also emit individual segments for FTS matching. |
| 714 | for part in split_compound(clean) { |
| 715 | if part.len() >= 3 |
| 716 | && !stop_words.contains(part.to_lowercase().as_str()) |
| 717 | && seen.insert(part.to_string()) |
| 718 | { |
| 719 | symbols.push(part.to_string()); |
| 720 | } |
| 721 | } |
| 722 | return; |
| 723 | } |
| 724 | |
| 725 | // CamelCase |
| 726 | if is_camel_case(clean) { |
| 727 | if !stop_words.contains(clean.to_lowercase().as_str()) && seen.insert(clean.to_string()) { |
| 728 | symbols.push(clean.to_string()); |
| 729 | } |
| 730 | // Also emit individual segments for FTS matching. |
| 731 | for part in split_compound(clean) { |
| 732 | if part.len() >= 3 |
| 733 | && !stop_words.contains(part.to_lowercase().as_str()) |
| 734 | && seen.insert(part.to_string()) |
| 735 | { |
| 736 | symbols.push(part.to_string()); |
| 737 | } |
| 738 | } |
no test coverage detected