returns true if one more multitoken is found
| 77 | } |
| 78 | //! returns true if one more multitoken is found |
| 79 | bool Next() { |
| 80 | if (Finished()) |
| 81 | return false; |
| 82 | |
| 83 | Tokens.clear(); |
| 84 | size_t i = First; |
| 85 | do { |
| 86 | const TCharSpan& s = Subtokens[i]; |
| 87 | if (!Tokens.empty() && Tokens.back().TokenDelim == TOKDELIM_NULL) { |
| 88 | TCharSpan& mark = Tokens.back(); |
| 89 | mark.Len += s.Len; |
| 90 | mark.SuffixLen = s.SuffixLen; |
| 91 | mark.Type = TOKEN_MARK; // change type |
| 92 | NlpType = NLP_MARK; |
| 93 | } else { |
| 94 | Y_ASSERT(Tokens.empty() || Tokens.back().Type == s.Type); |
| 95 | Tokens.push_back(s); |
| 96 | NlpType = (s.Type == TOKEN_WORD ? NLP_WORD : NLP_INTEGER); |
| 97 | } |
| 98 | } while (!BreakMultitoken(Subtokens, First, Last, i++, Tokens.size())); |
| 99 | Y_ASSERT(!Tokens.empty()); |
| 100 | |
| 101 | if (NlpType == NLP_INTEGER && Tokens.size() == 2) { |
| 102 | Y_ASSERT(Tokens[0].SuffixLen == 0 && Tokens[0].TokenDelim == TOKDELIM_DOT); // && Tokens[1].SuffixLen == 0); |
| 103 | NlpType = NLP_FLOAT; |
| 104 | TCharSpan& first = Tokens[0]; |
| 105 | const TCharSpan& second = Tokens[1]; |
| 106 | first.Len = second.EndPos() - first.Pos; |
| 107 | first.SuffixLen = second.SuffixLen; |
| 108 | first.Type = TOKEN_FLOAT; |
| 109 | first.TokenDelim = TOKDELIM_NULL; |
| 110 | Tokens.resize(1); |
| 111 | } |
| 112 | |
| 113 | Tokens.back().TokenDelim = TOKDELIM_NULL; // reset the last delimiter |
| 114 | First = i; |
| 115 | return true; |
| 116 | } |
| 117 | //! @note positions of subtokens of the original multitoken are not changed; |
| 118 | //! all tokens can have suffixes |
| 119 | const TTokenStructure& Get() const { |