| 110 | // Tokens.reserve(MAX_SUBTOKENS); don't reserve memory |
| 111 | } |
| 112 | void push_back(const TCharSpan& s) { |
| 113 | push_back(s.Pos, s.Len, s.Type, s.TokenDelim, s.Hyphen, s.SuffixLen, s.PrefixLen); |
| 114 | } |
| 115 | void push_back(size_t offset, size_t len, ETokenType type = TOKEN_WORD, ETokenDelim tokenDelim = TOKDELIM_NULL, EHyphenType hyphen = HYPHEN_NONE, |
| 116 | ui16 suffixLen = 0, ui16 prefixLen = 0) { |
| 117 | Y_ASSERT(type == TOKEN_WORD || type == TOKEN_NUMBER || type == TOKEN_MARK || type == TOKEN_FLOAT || type == TOKEN_MIXED); |