Function split_utf8_spans

src/framework/text/chunking.cpp:65–94

           token == u8"，" || token == u8"、" || token == u8"；" || token == u8"：";
}

bool is_tag_open(std::string_view token) {
    return token == "[" || token == "<";
}

bool is_tag_close(std::string_view token, std::string_view open) {
    return (open == "[" && token == "]") || (open == "<" && token == ">");
}

std::vector<Utf8Span> split_utf8_spans(std::string_view text, std::string_view label) {
    std::vector<Utf8Span> spans;
    spans.reserve(utf8_codepoint_count(text, label));
    for (size_t pos = 0; pos < text.size();) {
        const auto ch = static_cast<unsigned char>(text[pos]);
        size_t width = 0;
        if (ch <= 0x7FU) {
            width = 1;
        } else if ((ch & 0xE0U) == 0xC0U) {
            width = 2;
        } else if ((ch & 0xF0U) == 0xE0U) {
            width = 3;
        } else if ((ch & 0xF8U) == 0xF0U) {
            width = 4;
        } else {
            throw std::runtime_error(std::string(label) + " contains invalid UTF-8");
        }
        if (pos + width > text.size()) {
            throw std::runtime_error(std::string(label) + " contains truncated UTF-8");
        }
        for (size_t i = 1; i < width; ++i) {
            if (!is_utf8_continuation(static_cast<unsigned char>(text[pos + i]))) {
                throw std::runtime_error(std::string(label) + " contains invalid UTF-8 continuation byte");
            }