| 109 | } |
| 110 | } |
| 111 | void CorrectLastToken(TCharSpan& last) { |
| 112 | Y_ASSERT(Subtokens.size() == MAX_SUBTOKENS); |
| 113 | last = Subtokens.back(); |
| 114 | Subtokens.pop_back(); |
| 115 | // change delimiter (+) to suffix |
| 116 | TCharSpan& lasttok = Subtokens.back(); |
| 117 | // actually '+' should be added if subtoken has suffix '+' because '++' is valid suffix as well |
| 118 | if (lasttok.TokenDelim == TOKDELIM_PLUS && lasttok.SuffixLen == 0) |
| 119 | lasttok.SuffixLen = 1; |
| 120 | else if (lasttok.TokenDelim == TOKDELIM_AT_SIGN && last.PrefixLen == 0) |
| 121 | last.PrefixLen = 1; |
| 122 | lasttok.TokenDelim = TOKDELIM_NULL; |
| 123 | } |
| 124 | void Reset() { |
| 125 | Subtokens.clear(); |
| 126 | NlpType = NLP_WORD; |