Map a bare word to a placeholder when it looks like an identifier. Only length and shape are used — a token is not identified by its spelling, so nothing sensitive is retained in the grouping key.
(word: &str)
| 584 | /// Only length and shape are used — a token is not identified by its |
| 585 | /// spelling, so nothing sensitive is retained in the grouping key. |
| 586 | fn normalize_token(word: &str) -> String { |
| 587 | let has_digit = word.chars().any(|c| c.is_ascii_digit()); |
| 588 | let long = word.len() >= 16; |
| 589 | if has_digit && long { |
| 590 | "ID".to_string() |
| 591 | } else if has_digit && word.chars().all(|c| c.is_ascii_alphanumeric()) { |
| 592 | "N".to_string() |
| 593 | } else { |
| 594 | word.to_ascii_lowercase() |
| 595 | } |
| 596 | } |
| 597 | |
| 598 | fn truncate_detail(s: &str) -> String { |
| 599 | let flat = s.split_whitespace().collect::<Vec<_>>().join(" "); |
no test coverage detected