Function utf8_codepoints

src/models/miotts/session.cpp:272–308

}

std::vector<uint32_t> utf8_codepoints(std::string_view text, std::string_view label) {
    std::vector<uint32_t> out;
    out.reserve(engine::text::utf8_codepoint_count(text, label));
    for (size_t pos = 0; pos < text.size();) {
        const auto ch = static_cast<unsigned char>(text[pos]);
        size_t width = 0;
        uint32_t codepoint = 0;
        if (ch <= 0x7FU) {
            width = 1;
            codepoint = ch;
        } else if ((ch & 0xE0U) == 0xC0U) {
            width = 2;
            codepoint = ch & 0x1FU;
        } else if ((ch & 0xF0U) == 0xE0U) {
            width = 3;
            codepoint = ch & 0x0FU;
        } else if ((ch & 0xF8U) == 0xF0U) {
            width = 4;
            codepoint = ch & 0x07U;
        } else {
            throw std::runtime_error(std::string(label) + " contains invalid UTF-8");
        }
        if (pos + width > text.size()) {
            throw std::runtime_error(std::string(label) + " contains truncated UTF-8");
        }
        for (size_t i = 1; i < width; ++i) {
            const auto cont = static_cast<unsigned char>(text[pos + i]);
            if (!engine::text::is_utf8_continuation(cont)) {
                throw std::runtime_error(std::string(label) + " contains invalid UTF-8 continuation byte");
            }
            codepoint = (codepoint << 6U) | (cont & 0x3FU);
        }
        out.push_back(codepoint);
        pos += width;
    }
    return out;
}

bool is_japanese_codepoint(uint32_t codepoint) {
    return (codepoint >= 0x3040U && codepoint <= 0x309FU) ||