Method GetDelimiter

library/cpp/token/token_iterator.h:195–209

@note NLP type of token is NLP_MISCTEXT; prefixes always considered as "misctext"; suffixes of non-words considered as "misctext"; this function can be called after the last token as well, especially when the last non-word token has the suffix

    //!       this function can be called after the last token as well,
    //!       especially when the last non-word token has the suffix
    void GetDelimiter(TWideToken& tok) const {
        Y_ASSERT(!Tokens.empty()); // Next() must be called
        //Y_ASSERT(!Finished());
        const TCharSpan& prev = Tokens.back();
        size_t endpos = prev.EndPos();
        if (prev.Type == TOKEN_WORD) {
            endpos += prev.SuffixLen;
            if (!Finished() && Subtokens[First].PrefixLen)
                endpos += GetAdditionalSuffixLen();
        } else if (NlpType == NLP_INTEGER && !Finished() && Subtokens[First].PrefixLen)
            endpos += GetIntegerSuffixLen();
        tok.Token = Tok.Token + endpos;
        tok.Leng = (Finished() ? Tok.Leng : Tok.SubTokens[First].Pos) - endpos; // length can be equal to 0 in case v1.0 -> v 1.0
        tok.SubTokens.clear();
    }
};