Method AddToken

library/cpp/tokenizer/nlpparserbase.h:167–185

        CurCharSpan.Len += 1;
    }
    void AddToken() {
        Y_ASSERT(CurCharSpan.Len);
        Y_ASSERT(CurCharSpan.Type == TOKEN_WORD || CurCharSpan.Type == TOKEN_NUMBER);

        if (Current->GetSubtokenCount() == MAX_SUBTOKENS) {
            TCharSpan last; // for backward compatibility the last subtoken of the previous multitoken is popped and ...
            Current->CorrectLastToken(last);
            Tokens.push_back(TParserToken(last)); // ... added to the front of the new multitoken
            Current = &Tokens.back();             // new parser token, Current reassigned immediately after push_back()
        }

        Current->AddSubtoken(CurCharSpan, PrefixLen, PrefixChar, SuffixChar);

        CurCharSpan.Pos = 0;
        CurCharSpan.Len = 0; // it is checked in AddLastToken()
        PrefixLen = 0;
        PrefixChar = 0;
        SuffixChar = 0;
    }
    void AddIdeograph(size_t len) {
        Y_ASSERT(!CurCharSpan.Len && (len == 1 || len == 2));
        Current->AddIdeograph(len);