Function is_sentence_break

src/framework/text/chunking.cpp:47–50

    return token.size() == 1 && std::isspace(static_cast<unsigned char>(token.front())) != 0;
}

bool is_ascii_line_break(std::string_view token) noexcept {
    return token == "\n" || token == "\r";
}

bool is_horizontal_ascii_space(std::string_view token) noexcept {
    return token == " " || token == "\t" || token == "\f" || token == "\v";