Method MakeMultitokenEntry

library/cpp/tokenizer/nlpparser.cpp:82–123

}

template<> void TVersionedNlpParser<2>::MakeMultitokenEntry(TParserToken& token, const wchar16* entry) {
    size_t entryLen = token.GetLength();

    TTokenStructure subtokens;
    token.CorrectPositions();
    Y_ASSERT(token.GetLength() == entryLen); // check after the correction
    token.SwapSubtokens(subtokens);

    const wchar16* tokenText = entry;
    size_t tokenLen = entryLen;
    NLP_TYPE type = token.GetNlpType();
    wchar16 buffer[TOKEN_MAX_BUF];
    if (token.HasHyphen()) {
        type = PrepareMultitoken(subtokens, buffer, TOKEN_MAX_BUF, entry, tokenLen);
        Y_ASSERT(tokenLen <= TOKEN_MAX_LEN); // multitoken isn't longer than TOKEN_MAX_LEN
        tokenText = buffer;
    } else {
        // NLP_WORD, NLP_INTEGER, NLP_FLOAT, NLP_MARK are needed to be cut off
        Y_ASSERT(type == NLP_WORD || type == NLP_INTEGER || type == NLP_FLOAT || type == NLP_MARK);
        if (tokenLen > TOKEN_MAX_LEN) {
            // entry here always points to the original text
            CutTooLongMultitoken(subtokens, entry, tokenLen, entryLen, type);
            tokenText = entry; // in case if leading accents were removed...
        }
    }

    TWideToken multitoken; // don't call to constructor TWideToken(entry, leng)!
    multitoken.Token = tokenText;
    multitoken.Leng = tokenLen;
    multitoken.SubTokens.swap(subtokens);
    Y_ASSERT(CheckMultitoken(multitoken));

    const size_t totalLen = entryLen + GetExtraLen(entry, entryLen);

    Y_ASSERT(multitoken.SubTokens.size());
    if (BackwardCompatible) {
        PassBackwardCompatibleToken(multitoken, type, totalLen);
    } else {
        SentBreakFilter.OnToken(multitoken, type);
        TokenHandler.OnToken(multitoken, totalLen, type);
    }
}

void TVersionedNlpParser<3>::MakeMultitokenEntry(TParserToken& token, const wchar16* entry) {
    size_t entryLen = token.GetLength();