Function SplitByDelimiter

library/cpp/text_processing/tokenizer/tokenizer.cpp:189–209

template <typename StringType>
static void SplitByDelimiter(
    TStringBuf inputString,
    const TString& delimiter,
    bool splitBySet,
    bool skipEmpty,
    TVector<StringType>* tokens
) {
    if (splitBySet) {
        if (skipEmpty) {
            *tokens = StringSplitter(inputString).SplitBySet(delimiter.c_str()).SkipEmpty();
        } else {
            *tokens = StringSplitter(inputString).SplitBySet(delimiter.c_str());
        }
    } else {
        if (skipEmpty) {
            *tokens = StringSplitter(inputString).SplitByString(delimiter).SkipEmpty();
        } else {
            *tokens = StringSplitter(inputString).SplitByString(delimiter);
        }
    }
}

template <typename StringType>
static void FilterNumbers(TVector<StringType>* tokens) {