Method GetMultitoken

library/cpp/token/token_iterator.h:127–154

the first subtoken of multitoken has position equal to 0 @note only word tokens can have suffixes

    //! the first subtoken of multitoken has position equal to 0
    //! @note only word tokens can have suffixes
    void GetMultitoken(TWideToken& tok) const {
        Y_ASSERT(!Tokens.empty());
        tok.SubTokens = Tokens;
        TTokenStructure& subtokens = tok.SubTokens;
        const TCharSpan& first = subtokens[0];
        TCharSpan& last = subtokens.back();
        tok.Token = Tok.Token + first.Pos;
        if (last.Type == TOKEN_WORD) {
            tok.Leng = last.EndPos() + last.SuffixLen - first.Pos;
            if (!Finished() && Subtokens[First].PrefixLen) {
                const ui16 suffixLen = GetAdditionalSuffixLen();
                tok.Leng += suffixLen;
                last.SuffixLen += suffixLen;
            }
        } else {
            tok.Leng = last.EndPos() - first.Pos;
            last.SuffixLen = 0;
            if (NlpType == NLP_INTEGER && !Finished() && Subtokens[First].PrefixLen) {
                const ui16 suffixLen = GetIntegerSuffixLen();
                tok.Leng += suffixLen;
                last.SuffixLen = suffixLen;
            }
        }
        subtokens[0].PrefixLen = 0;
        const size_t diff = first.Pos;
        for (auto& subtoken : subtokens)
            subtoken.Pos -= diff;
    }
    ui16 GetAdditionalSuffixLen() const {
        const TCharSpan& origtok = Subtokens[First - 1];
        Y_ASSERT(origtok.Type == TOKEN_WORD && !Finished() && Subtokens[First].PrefixLen);