Method AddIdeograph

library/cpp/tokenizer/nlpparserbase.h:75–79

        Y_ASSERT(NlpType == NLP_WORD || NlpType == NLP_MARK || NlpType == NLP_INTEGER);
    }
    void AddIdeograph(size_t len) {
        Y_ASSERT(Subtokens.empty());
        Subtokens.push_back(0, len, TOKEN_WORD);
        NlpType = NLP_WORD;
    }
    void SwapSubtokens(TTokenStructure& other) {
        Subtokens.swap(other);
    }