Detect language with a lightweight heuristic. If the input contains any CJK character, we treat it as Chinese; otherwise default to English. This avoids external dependencies and keeps decisions local and deterministic.
(text: &str)
| 134 | /// default to English. This avoids external dependencies and keeps decisions |
| 135 | /// local and deterministic. |
| 136 | pub fn detect_language(text: &str) -> TextEmbeddingLanguage { |
| 137 | if text.chars().any(is_cjk) { |
| 138 | TextEmbeddingLanguage::Chinese |
| 139 | } else { |
| 140 | TextEmbeddingLanguage::English |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | fn is_cjk(ch: char) -> bool { |
| 145 | matches!( |
no outgoing calls