Function next_utf8_codepoint

src/models/voxcpm2/tokenizer_text.cpp:16–49

namespace {

uint32_t next_utf8_codepoint(std::string_view text, size_t & offset) {
    if (offset >= text.size()) {
        throw std::runtime_error("VoxCPM2 tokenizer UTF-8 offset is out of range");
    }
    const unsigned char first = static_cast<unsigned char>(text[offset]);
    uint32_t codepoint = 0;
    size_t len = 1;
    if ((first & 0x80U) == 0) {
        codepoint = first;
    } else if ((first & 0xE0U) == 0xC0U) {
        len = 2;
        codepoint = first & 0x1FU;
    } else if ((first & 0xF0U) == 0xE0U) {
        len = 3;
        codepoint = first & 0x0FU;
    } else if ((first & 0xF8U) == 0xF0U) {
        len = 4;
        codepoint = first & 0x07U;
    } else {
        throw std::runtime_error("VoxCPM2 tokenizer encountered invalid UTF-8");
    }
    if (offset + len > text.size()) {
        throw std::runtime_error("VoxCPM2 tokenizer encountered truncated UTF-8");
    }
    for (size_t i = 1; i < len; ++i) {
        const unsigned char ch = static_cast<unsigned char>(text[offset + i]);
        if ((ch & 0xC0U) != 0x80U) {
            throw std::runtime_error("VoxCPM2 tokenizer encountered invalid UTF-8 continuation");
        }
        codepoint = (codepoint << 6U) | (ch & 0x3FU);
    }
    offset += len;
    return codepoint;
}

std::vector<std::string> utf8_codepoints(std::string_view text) {
    std::vector<std::string> out;