Method OnToken

library/cpp/tokenizer/split.cpp:34–40

        }

        void OnToken(const TWideToken& token, size_t, NLP_TYPE type) override {
            CurToken += token.Text();

            if (type == NLP_SENTBREAK || type == NLP_PARABREAK) {
                Flush();
            }
        }

        void Flush() {
            if (!CurToken.empty()) {