Function normalize_chatterbox_tts_text

src/models/chatterbox/text_tokenizer.cpp:335–396

}

std::string normalize_chatterbox_tts_text(const std::string & text) {
    if (text.empty()) {
        return "You need to add some text for me to talk.";
    }

    std::string normalized = text;
    if (!normalized.empty() && std::islower(static_cast<unsigned char>(normalized.front())) != 0) {
        normalized.front() = static_cast<char>(std::toupper(static_cast<unsigned char>(normalized.front())));
    }

    std::string compact;
    compact.reserve(normalized.size());
    bool previous_space = false;
    for (char ch : normalized) {
        if (std::isspace(static_cast<unsigned char>(ch)) != 0) {
            if (!previous_space) {
                compact.push_back(' ');
                previous_space = true;
            }
            continue;
        }
        compact.push_back(ch);
        previous_space = false;
    }
    normalized = trim_spaces(std::move(compact));

    const std::vector<std::pair<std::string, std::string>> replacements = {
        {"...", ", "},
        {"…", ", "},
        {":", ","},
        {" - ", ", "},
        {";", ", "},
        {"—", "-"},
        {"–", "-"},
        {" ,", ","},
        {"“", "\""},
        {"”", "\""},
        {"‘", "'"},
        {"’", "'"},
    };
    for (const auto & [old_text, new_text] : replacements) {
        size_t search = 0;
        while ((search = normalized.find(old_text, search)) != std::string::npos) {
            normalized.replace(search, old_text.size(), new_text);
            search += new_text.size();
        }
    }

    normalized = trim_spaces(std::move(normalized));
    const std::vector<std::string> sentence_enders = {".", "!", "?", "-", ",", "、", "，", "。", "？", "！"};
    const bool has_sentence_ender = std::any_of(
        sentence_enders.begin(),
        sentence_enders.end(),
        [&](const std::string & ending) {
            return normalized.size() >= ending.size() &&
                normalized.compare(normalized.size() - ending.size(), ending.size(), ending) == 0;
        });
    if (normalized.empty() || !has_sentence_ender) {