Method PassBackwardCompatibleToken

library/cpp/tokenizer/nlpparser.cpp:241–286

}

void TNlpParser::PassBackwardCompatibleToken(const TWideToken& multitoken, NLP_TYPE type, size_t totalLen) {
    if (multitoken.SubTokens.size() == 1) {
        const TCharSpan& subtok = multitoken.SubTokens[0];
        TWideToken tok;
        if (subtok.PrefixLen) {
            tok.Token = multitoken.Token;
            tok.Leng = subtok.PrefixLen;
            SentBreakFilter.OnToken(tok, NLP_MISCTEXT);
            TokenHandler.OnToken(tok, tok.Leng, NLP_MISCTEXT);
        }

        const ui16 prefixLen = subtok.PrefixLen;
        tok.Token = multitoken.Token + prefixLen;
        tok.Leng = multitoken.Leng - prefixLen;
        tok.SubTokens.push_back(subtok);
        tok.SubTokens[0].PrefixLen = 0;
        tok.SubTokens[0].Pos -= prefixLen;

        SentBreakFilter.OnToken(tok, type);
        TokenHandler.OnToken(tok, tok.Leng + (totalLen - multitoken.Leng), type);
        // suffix after alone number is kept, for example: 18+ -> [18+]
        // if number with suffix is part of multitoken then suffix will be removed: 16+/18+ -> [16]+/[18]+
        // see also tokenizer_ut.cpp
    } else {
        TWideToken tok;
        TTokenIterator it(multitoken);
        it.GetPrefix(tok); // prefix of the first token
        if (tok.Leng) {
            SentBreakFilter.OnToken(tok, NLP_MISCTEXT);
            TokenHandler.OnToken(tok, tok.Leng, NLP_MISCTEXT);
        }
        while (it.Next()) {
            it.GetMultitoken(tok);
            SentBreakFilter.OnToken(tok, it.GetNlpType());
            if (!it.Finished())
                TokenHandler.OnToken(tok, tok.Leng, it.GetNlpType());
            else
                TokenHandler.OnToken(tok, tok.Leng + (totalLen - multitoken.Leng), it.GetNlpType());
            it.GetDelimiter(tok);
            if (tok.Leng) {
                SentBreakFilter.OnToken(tok, NLP_MISCTEXT);
                TokenHandler.OnToken(tok, tok.Leng, NLP_MISCTEXT);
            }
        }
    }
}

void TNlpParser::MakeEntry(const wchar16* entry, size_t entryLen, NLP_TYPE type) {
    TWideToken token; // don't call to constructor TWideToken(entry, leng)!