| 270 | } |
| 271 | |
| 272 | inline wchar16 GetRightTokenDelim(const TWideToken& tok, size_t index) { |
| 273 | const TTokenStructure& subtokens = tok.SubTokens; |
| 274 | Y_ASSERT(index < subtokens.size()); |
| 275 | if (index + 1 >= subtokens.size()) |
| 276 | return 0; |
| 277 | const TCharSpan& subtok = subtokens[index]; |
| 278 | const size_t delimPos = subtok.EndPos() + subtok.SuffixLen; |
| 279 | const TCharSpan& next = subtokens[index + 1]; |
| 280 | return (delimPos == next.Pos - next.PrefixLen ? 0 : tok.Token[delimPos]); |
| 281 | } |
| 282 | |
| 283 | //! @note yc_minus (0x2D) can be 0xB7 that is an ASCII character as well |
| 284 | inline char GetTokenDelimChar(ETokenDelim delim) { |