Function split_word_ranges

src/framework/text/chunking.cpp:96–123

        for (size_t i = 1; i < width; ++i) {
            if (!is_utf8_continuation(static_cast<unsigned char>(text[pos + i]))) {
                throw std::runtime_error(std::string(label) + " contains invalid UTF-8 continuation byte");
            }
        }
        spans.push_back({pos, pos + width, text.substr(pos, width)});
        pos += width;
    }
    return spans;
}

std::vector<WordRange> split_word_ranges(const std::vector<Utf8Span> & spans) {
    std::vector<WordRange> words;
    size_t span_pos = 0;
    while (span_pos < spans.size()) {
        while (span_pos < spans.size() && is_ascii_space(spans[span_pos].text)) {
            ++span_pos;
        }
        if (span_pos >= spans.size()) {
            break;
        }
        const size_t word_start = span_pos;
        size_t word_end = span_pos + 1;
        while (word_end < spans.size() && !is_ascii_space(spans[word_end].text)) {
            ++word_end;
        }
        const auto last = spans[word_end - 1].text;
        words.push_back({
            word_start,
            word_end,
            spans[word_start].start,
            spans[word_end - 1].end,
            is_sentence_break(last),