Function ExtractWideTokenRange

library/cpp/token/token_util.cpp:84–97

}

TWideToken ExtractWideTokenRange(const TWideToken& tok, size_t start, size_t end) {
    Y_ASSERT(start < tok.SubTokens.size());
    Y_ASSERT(end < tok.SubTokens.size());

    TWideToken newToken;
    const size_t offset = GetSubTokenOffset(tok, start);
    newToken.Token = tok.Token + offset;
    newToken.Leng = tok.SubTokens[end].EndPos() + tok.SubTokens[end].SuffixLen - offset;
    for (size_t j = start; j <= end; ++j) {
        newToken.SubTokens.push_back(tok.SubTokens[j]);
        newToken.SubTokens.back().Pos -= offset;
    }
    return newToken;
}