Function GetSubTokenOffset

library/cpp/token/token_util.h:24–27

bool CheckWideTokenReqSplit(const TTokenStructure& subtokens, size_t pos);

inline size_t GetSubTokenOffset(const TWideToken& tok, size_t subToken) {
    Y_ASSERT(subToken < tok.SubTokens.size());
    return tok.SubTokens[subToken].Pos - tok.SubTokens[subToken].PrefixLen;
}

// Create a new wide-token with the specified inclusive [start, end] sub-token range
TWideToken ExtractWideTokenRange(const TWideToken& tok, size_t start, size_t end);