Method AddIdeograph

library/cpp/tokenizer/nlpparserbase.h:186–189

        SuffixChar = 0;
    }
    void AddIdeograph(size_t len) {
        Y_ASSERT(!CurCharSpan.Len && (len == 1 || len == 2));
        Current->AddIdeograph(len);
    }
    void AddLastToken(const wchar16* tokstart, const wchar16* tokend) {
        // - CurCharSpan.Len assigned to 0 in AddToken() because in case of multitoken with '.' at the end, for
        //   example: " well-formed. " parser already called to %add_token because '.' can be delimiter of the next token