replresents single multitoken in the collection of TNlpParserBase
| 6 | |
| 7 | // replresents single multitoken in the collection of TNlpParserBase |
| 8 | class TParserToken { |
| 9 | TTokenStructure Subtokens; |
| 10 | NLP_TYPE NlpType; // type of multitoken |
| 11 | bool Hyphen; |
| 12 | |
| 13 | public: |
| 14 | TParserToken() |
| 15 | : NlpType(NLP_WORD) |
| 16 | , Hyphen(false) |
| 17 | { |
| 18 | } |
| 19 | |
| 20 | explicit TParserToken(const TCharSpan& subtok) |
| 21 | : NlpType(NLP_WORD) |
| 22 | , Hyphen(false) |
| 23 | { |
| 24 | Subtokens.push_back(subtok); |
| 25 | } |
| 26 | bool HasHyphen() const { |
| 27 | return Hyphen; |
| 28 | } |
| 29 | void SetHyphen(EHyphenType type) { |
| 30 | Y_ASSERT(!Subtokens.empty()); |
| 31 | Subtokens.back().Hyphen = type; |
| 32 | Hyphen = true; |
| 33 | } |
| 34 | void SetTokenDelim(ETokenDelim delim) { |
| 35 | Y_ASSERT(!Subtokens.empty()); |
| 36 | Subtokens.back().TokenDelim = delim; |
| 37 | } |
| 38 | NLP_TYPE GetNlpType() const { |
| 39 | return NlpType; |
| 40 | } |
| 41 | size_t GetSubtokenCount() const { |
| 42 | return Subtokens.size(); |
| 43 | } |
| 44 | size_t GetStart() const { |
| 45 | Y_ASSERT(!Subtokens.empty()); |
| 46 | return Subtokens[0].Pos - Subtokens[0].PrefixLen; |
| 47 | } |
| 48 | size_t GetEnd() const { |
| 49 | Y_ASSERT(!Subtokens.empty()); |
| 50 | return Subtokens.back().EndPos() + Subtokens.back().SuffixLen; |
| 51 | } |
| 52 | size_t GetLength() const { |
| 53 | return GetEnd() - GetStart(); |
| 54 | } |
| 55 | // allowed suffixes: "+", "++" and "#" |
| 56 | void AddSubtoken(const TCharSpan& span, size_t prefixLen, wchar16 prefixChar, wchar16 suffixChar) { |
| 57 | Y_ASSERT(Subtokens.size() < MAX_SUBTOKENS); |
| 58 | |
| 59 | if (Subtokens.empty()) { |
| 60 | NlpType = (span.Type == TOKEN_WORD ? NLP_WORD : NLP_INTEGER); |
| 61 | } else { |
| 62 | if (NlpType != NLP_MARK && span.Type != Subtokens.back().Type) |
| 63 | NlpType = NLP_MARK; |
| 64 | } |
| 65 |