Method encode

src/models/voxcpm2/tokenizer_text.cpp:288–321

}

std::vector<int32_t> VoxCPM2TextTokenizer::encode(const std::string & text) const {
    std::vector<int32_t> ids;
    for (size_t i = 0; i < text.size();) {
        const auto special_it = std::find_if(
            impl_->special_tokens.begin(),
            impl_->special_tokens.end(),
            [&](const auto & item) { return starts_with_at(text, i, item.first); });
        if (special_it != impl_->special_tokens.end()) {
            impl_->append_expanded_id(ids, special_it->second);
            i += special_it->first.size();
            continue;
        }

        size_t next_special = text.size();
        for (const auto & [special, _] : impl_->special_tokens) {
            const size_t pos = text.find(special, i);
            if (pos != std::string::npos) {
                next_special = std::min(next_special, pos);
            }
        }
        const std::string normalized = normalize_text(std::string_view(
            text.data() + static_cast<std::ptrdiff_t>(i),
            next_special - i));
        for (const auto & bpe_token : impl_->bpe(normalized)) {
            const auto vocab_it = impl_->vocab.find(bpe_token);
            if (vocab_it == impl_->vocab.end()) {
                throw std::runtime_error("VoxCPM2 tokenizer produced token not present in vocab: " + bpe_token);
            }
            impl_->append_expanded_id(ids, vocab_it->second);
        }
        i = next_special;
    }
    return ids;
}

VoxCPM2TextPrompt VoxCPM2TextTokenizer::build_prompt(const std::string & text) const {
    if (text.empty()) {