Function is_clause_break

src/framework/text/chunking.cpp:52–55

}

bool is_horizontal_ascii_space(std::string_view token) noexcept {
    return token == " " || token == "\t" || token == "\f" || token == "\v";
}

bool is_sentence_break(std::string_view token) {
    return token == "." || token == "!" || token == "?" ||
           token == u8"。" || token == u8"！" || token == u8"？";