Function build_cjk_split_map

src/models/voxcpm2/tokenizer_text.cpp:250–269

}

void build_cjk_split_map(VoxCPM2TextTokenizer::Impl & impl) {
    for (const auto & [id, token] : impl.id_to_token) {
        const std::string clean = strip_sentencepiece_prefix(token);
        if (!is_pure_multichar_cjk(clean)) {
            continue;
        }
        std::vector<int32_t> char_ids;
        for (const auto & ch : utf8_codepoints(clean)) {
            const auto it = impl.vocab.find(ch);
            if (it == impl.vocab.end()) {
                char_ids.clear();
                break;
            }
            char_ids.push_back(it->second);
        }
        if (!char_ids.empty()) {
            impl.cjk_split_map.emplace(id, std::move(char_ids));
        }
    }
}

std::shared_ptr<const VoxCPM2TextTokenizer::Impl> load_impl(const VoxCPM2Assets & assets) {
    auto impl = std::make_shared<VoxCPM2TextTokenizer::Impl>();