Function split_text_chunks_default

src/framework/text/chunking.cpp:296–361

    if (close == std::string::npos || close + 1 >= text.size()) {
        return std::nullopt;
    }
    const std::string prefix = engine::io::trim_ascii_whitespace(text.substr(0, close + 1));
    const std::string body = engine::io::trim_ascii_whitespace(text.substr(close + 1));
    if (prefix.empty() || body.empty()) {
        return std::nullopt;
    }
    return std::make_pair(prefix, body);
}

std::vector<std::string> split_text_chunks_default(
    std::string_view text,
    int64_t codepoint_budget) {
    if (codepoint_budget <= 0) {
        throw std::runtime_error("text chunk budget must be positive");
    }
    const std::string trimmed = engine::io::trim_ascii_whitespace(std::string(text));
    if (trimmed.empty()) {
        return {};
    }
    const auto spans = split_utf8_spans(trimmed, "text chunk");
    if (static_cast<int64_t>(spans.size()) <= codepoint_budget) {
        return {trimmed};
    }

    const auto words = split_word_ranges(spans);
    if (words.empty()) {
        return {};
    }

    std::vector<std::string> chunks;
    size_t word_start = 0;
    while (word_start < words.size()) {
        size_t hard_end = word_start;
        while (hard_end < words.size()) {
            const size_t chunk_span_start = words[word_start].span_start;
            const size_t candidate_span_end = words[hard_end].span_end;
            const auto chunk_codepoints = static_cast<int64_t>(candidate_span_end - chunk_span_start);
            if (chunk_codepoints > codepoint_budget) {
                break;
            }
            ++hard_end;
        }

        if (hard_end == word_start) {
            hard_end = word_start + 1;
        }

        size_t chunk_end = hard_end;
        if (hard_end < words.size() && hard_end > word_start + 1) {
            for (size_t i = hard_end; i > word_start + 1; --i) {
                if (words[i - 1].sentence_break) {
                    chunk_end = i;
                    break;
                }
            }
            if (chunk_end == hard_end) {
                for (size_t i = hard_end; i > word_start + 1; --i) {
                    if (words[i - 1].clause_break) {