| 286 | } |
| 287 | |
| 288 | void TNlpParser::MakeEntry(const wchar16* entry, size_t entryLen, NLP_TYPE type) { |
| 289 | TWideToken token; // don't call to constructor TWideToken(entry, leng)! |
| 290 | token.Token = entry; |
| 291 | token.Leng = entryLen; |
| 292 | |
| 293 | const size_t totalLen = entryLen + GetExtraLen(entry, entryLen); |
| 294 | |
| 295 | SentBreakFilter.OnToken(token, type); |
| 296 | TokenHandler.OnToken(token, totalLen, type); |
| 297 | } |
| 298 | |
| 299 | void TVersionedNlpParser<3>::MakeEntry(const wchar16* entry, size_t entryLen, NLP_TYPE type) { |
| 300 | if (KeepedPotentialPrefix) { |