allowed suffixes: "+", "++" and "#"
| 54 | } |
| 55 | // allowed suffixes: "+", "++" and "#" |
| 56 | void AddSubtoken(const TCharSpan& span, size_t prefixLen, wchar16 prefixChar, wchar16 suffixChar) { |
| 57 | Y_ASSERT(Subtokens.size() < MAX_SUBTOKENS); |
| 58 | |
| 59 | if (Subtokens.empty()) { |
| 60 | NlpType = (span.Type == TOKEN_WORD ? NLP_WORD : NLP_INTEGER); |
| 61 | } else { |
| 62 | if (NlpType != NLP_MARK && span.Type != Subtokens.back().Type) |
| 63 | NlpType = NLP_MARK; |
| 64 | } |
| 65 | |
| 66 | Subtokens.push_back(span); |
| 67 | Subtokens.back().PrefixLen = prefixLen; |
| 68 | |
| 69 | const size_t n = Subtokens.size(); |
| 70 | if (n > 1) |
| 71 | CorrectDelimiters(Subtokens[n - 2], suffixChar, Subtokens[n - 1], prefixChar); |
| 72 | |
| 73 | Y_ASSERT(NlpType == NLP_WORD || NlpType == NLP_MARK || NlpType == NLP_INTEGER); |
| 74 | } |
| 75 | void AddIdeograph(size_t len) { |
| 76 | Y_ASSERT(Subtokens.empty()); |
| 77 | Subtokens.push_back(0, len, TOKEN_WORD); |