Function utf8_codepoints

src/models/voxcpm2/tokenizer_text.cpp:51–59

}

std::vector<std::string> utf8_codepoints(std::string_view text) {
    std::vector<std::string> out;
    for (size_t offset = 0; offset < text.size();) {
        const size_t start = offset;
        (void) next_utf8_codepoint(text, offset);
        out.emplace_back(text.substr(start, offset - start));
    }
    return out;
}

std::string normalize_text(std::string_view text) {
    const std::string space = "\xE2\x96\x81";