| 197 | } |
| 198 | |
| 199 | void TNlpParser::CutTooLongMultitoken(TTokenStructure& subtokens, const wchar16*& entry, size_t& leng, size_t& origleng, NLP_TYPE& type) { |
| 200 | Y_ASSERT(leng > TOKEN_MAX_LEN); |
| 201 | if (type == NLP_WORD || type == NLP_INTEGER || type == NLP_MARK) { |
| 202 | // if too many accent symbols are in the beginning of the token (the number is greater than TOKEN_MAX_LEN) |
| 203 | // TODO: remove accents before tokenization |
| 204 | const ptrdiff_t n = FindNonAccent(entry, leng) - entry; |
| 205 | Y_ASSERT(n >= 0); |
| 206 | |
| 207 | // NLP_WORD contains words only, NLP_INTEGER - integers only, NLP_MARK - words and integers |
| 208 | Y_ASSERT(!subtokens.empty()); |
| 209 | |
| 210 | if (n > 0) { |
| 211 | const TWideToken miscText(entry, n); // the first part containing accents only |
| 212 | TokenHandler.OnToken(miscText, n, NLP_MISCTEXT); |
| 213 | origleng -= n; |
| 214 | entry += n; |
| 215 | leng = AdjustSubtokens(subtokens, n, TOKEN_MAX_LEN); |
| 216 | } else |
| 217 | leng = AdjustSubtokens(subtokens, TOKEN_MAX_LEN); |
| 218 | |
| 219 | // correct NLP type |
| 220 | if (type == NLP_MARK) { |
| 221 | Y_ASSERT(!subtokens.empty()); |
| 222 | ETokenType tokType = subtokens[0].Type; |
| 223 | Y_ASSERT(tokType == TOKEN_WORD || tokType == TOKEN_NUMBER); |
| 224 | for (size_t i = 1; i < subtokens.size(); ++i) { |
| 225 | if (subtokens[i].Type != tokType) { |
| 226 | tokType = TOKEN_MARK; |
| 227 | break; |
| 228 | } |
| 229 | } |
| 230 | if (tokType != TOKEN_MARK) |
| 231 | type = (tokType == TOKEN_WORD ? NLP_WORD : NLP_INTEGER); |
| 232 | } |
| 233 | } else { |
| 234 | // no processing of the case when point of a NLP_FLOAT token is cut off (position of the |
| 235 | // point character is greater than TOKEN_MAX_LEN) and token actually will be integer |
| 236 | Y_ASSERT(subtokens.empty()); |
| 237 | leng = TOKEN_MAX_LEN; |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | void TNlpParser::PassBackwardCompatibleToken(const TWideToken& multitoken, NLP_TYPE type, size_t totalLen) { |
| 242 | if (multitoken.SubTokens.size() == 1) { |
nothing calls this directly
no test coverage detected