Convert text to a sequence of token IDs.
(&self, text: &str)
| 92 | |
| 93 | /// Convert text to a sequence of token IDs. |
| 94 | pub fn tokenize(&self, text: &str) -> Result<Vec<u32>> { |
| 95 | let text = text.to_lowercase(); |
| 96 | let mut tokens = Vec::new(); |
| 97 | |
| 98 | // Split into words and punctuation |
| 99 | let words: Vec<&str> = text.split_whitespace().collect(); |
| 100 | |
| 101 | for (i, word) in words.iter().enumerate() { |
| 102 | if i > 0 { |
| 103 | if let Some(sp) = self.space_id { |
| 104 | tokens.push(sp); |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | // Strip punctuation from edges |
| 109 | let clean = word.trim_matches(|c: char| !c.is_alphanumeric()); |
| 110 | if clean.is_empty() { |
| 111 | // Pure punctuation — look up directly |
| 112 | let punct_tokens = self.map_ipa_to_tokens(word); |
| 113 | tokens.extend(punct_tokens); |
| 114 | continue; |
| 115 | } |
| 116 | |
| 117 | // Look up in dictionary |
| 118 | if let Some(ipa) = self.ipa_dict.get(clean) { |
| 119 | let ipa_tokens = self.map_ipa_to_tokens(ipa); |
| 120 | tokens.extend(ipa_tokens); |
| 121 | } else { |
| 122 | // OOV fallback: rule-based letter-to-IPA |
| 123 | let ipa = self.rules_fallback(clean); |
| 124 | let ipa_tokens = self.map_ipa_to_tokens(&ipa); |
| 125 | tokens.extend(ipa_tokens); |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | Ok(tokens) |
| 130 | } |
| 131 | |
| 132 | /// Map an IPA string to token IDs by greedily matching the longest token. |
| 133 | fn map_ipa_to_tokens(&self, ipa: &str) -> Vec<u32> { |