| 69 | } |
| 70 | |
| 71 | void AddToken() { |
| 72 | Y_ASSERT(CurCharSpan.Len); |
| 73 | Y_ASSERT(CurCharSpan.Type == TOKEN_WORD || CurCharSpan.Type == TOKEN_NUMBER); |
| 74 | |
| 75 | TTokenStructure& tokens = Multitoken.SubTokens; |
| 76 | |
| 77 | // @todo if number of tokens is greater than 64 then the last token can consist of numbers, letters and delimiters... |
| 78 | tokens.push_back(CurCharSpan); |
| 79 | |
| 80 | const size_t n = tokens.size(); |
| 81 | if (n > 1) |
| 82 | CorrectDelimiters(tokens[n - 2], SuffixChar, tokens[n - 1], PrefixChar); |
| 83 | |
| 84 | CurCharSpan.Pos = 0; |
| 85 | CurCharSpan.Len = 0; // it is checked in AddLastToken() |
| 86 | CurCharSpan.PrefixLen = 0; |
| 87 | PrefixChar = 0; |
| 88 | SuffixChar = 0; |
| 89 | } |
| 90 | |
| 91 | void AddIdeograph(size_t len) { |
| 92 | Y_ASSERT(!CurCharSpan.Len && (len == 1 || len == 2)); |
nothing calls this directly
no test coverage detected