Method OnToken

library/cpp/tokenizer/split.cpp:14–20

        }

        void OnToken(const TWideToken& token, size_t, NLP_TYPE type) override {
            if (!Params.HandledMask.SafeTest(type)) {
                return;
            }

            Tokens->push_back(TUtf16String(token.Token, token.Leng));
        }

    private:
        TVector<TUtf16String>* Tokens;