| 269 | } |
| 270 | |
| 271 | std::shared_ptr<const VoxCPM2TextTokenizer::Impl> load_impl(const VoxCPM2Assets & assets) { |
| 272 | auto impl = std::make_shared<VoxCPM2TextTokenizer::Impl>(); |
| 273 | load_tokenizer_json(assets.resources.require_file("tokenizer_json"), *impl); |
| 274 | load_special_tokens(assets.resources.require_file("tokenizer_config"), *impl); |
| 275 | build_cjk_split_map(*impl); |
| 276 | return impl; |
| 277 | } |
| 278 | |
| 279 | } // namespace |
| 280 |
no test coverage detected