Method ProcessMultitoken

library/cpp/tokenizer/nlpparser.cpp:61–80

}

void TNlpParser::ProcessMultitoken(const wchar16* ts, const wchar16* te) {
    Base.AddLastToken(ts, te);
    const wchar16* p = ts;
    size_t prevEnd = 0;
    size_t tokenCount = Base.GetTokenCount();
    for (size_t i = 0; i < tokenCount; ++i) {
        TParserToken& tok = Base.GetToken(i);
        if (i) {
            size_t n = tok.GetStart() - prevEnd;
            if (n) {
                MakeEntry(p, n, NLP_MISCTEXT);
                p += n;
            }
        }
        prevEnd = tok.GetEnd();
        MakeMultitokenEntry(tok, p);
        p = ts + prevEnd;
    }
    Base.ResetTokens();
}

template<> void TVersionedNlpParser<2>::MakeMultitokenEntry(TParserToken& token, const wchar16* entry) {
    size_t entryLen = token.GetLength();