Function normalize_text

src/models/voxcpm2/tokenizer_text.cpp:61–72

}

std::string normalize_text(std::string_view text) {
    const std::string space = "\xE2\x96\x81";
    std::string out = space;
    for (char ch : text) {
        if (ch == ' ') {
            out += space;
        } else {
            out.push_back(ch);
        }
    }
    return out;
}

std::string byte_fallback_token(unsigned char byte) {
    constexpr char kHex[] = "0123456789ABCDEF";