Position of a single token in the word
| 38 | |
| 39 | // Position of a single token in the word |
| 40 | struct TCharSpan { |
| 41 | TCharSpan() |
| 42 | : Pos(0) |
| 43 | , Len(0) |
| 44 | , PrefixLen(0) |
| 45 | , SuffixLen(0) |
| 46 | , Type(TOKEN_WORD) |
| 47 | , Hyphen(HYPHEN_NONE) |
| 48 | , TokenDelim(TOKDELIM_NULL) |
| 49 | { |
| 50 | } |
| 51 | |
| 52 | TCharSpan(size_t offset, size_t len, ETokenType type = TOKEN_WORD, ETokenDelim tokenDelim = TOKDELIM_NULL, EHyphenType hyphen = HYPHEN_NONE, |
| 53 | ui16 suffixLen = 0, ui16 prefixLen = 0) |
| 54 | : Pos(offset) |
| 55 | , Len(len) |
| 56 | , PrefixLen(prefixLen) |
| 57 | , SuffixLen(suffixLen) |
| 58 | , Type(type) |
| 59 | , Hyphen(hyphen) |
| 60 | , TokenDelim(tokenDelim) |
| 61 | { |
| 62 | } |
| 63 | |
| 64 | size_t Pos; //!< offset from the beginning of the word to the start of the token |
| 65 | size_t Len; //!< token length, it does not include suffix length |
| 66 | ui16 PrefixLen; //!< Len doesn't include PrefixLen |
| 67 | ui16 SuffixLen; //!< suffix length |
| 68 | ETokenType Type; //!< type of this token |
| 69 | EHyphenType Hyphen; //!< hyphenation after the token, it should be verified by the lemmer and concatenated if the word is correct |
| 70 | ETokenDelim TokenDelim; //!< if true then apostrophe or minus follows this token |
| 71 | |
| 72 | //! @note end position does not include suffix length |
| 73 | inline size_t EndPos() const { |
| 74 | return Pos + Len; |
| 75 | } |
| 76 | |
| 77 | bool operator==(const TCharSpan& span) const { |
| 78 | return ((span.Pos == Pos) && (span.Len == Len) && |
| 79 | (span.PrefixLen == PrefixLen) && (span.SuffixLen == SuffixLen) && |
| 80 | (span.Type == Type) && (span.Hyphen == Hyphen) && (span.TokenDelim == TokenDelim)); |
| 81 | } |
| 82 | |
| 83 | size_t Hash() const { |
| 84 | size_t result = Pos; |
| 85 | result = CombineHashes(result, Len); |
| 86 | result = CombineHashes(result, size_t(PrefixLen)); |
| 87 | result = CombineHashes(result, size_t(SuffixLen)); |
| 88 | result = CombineHashes(result, size_t(Type)); |
| 89 | result = CombineHashes(result, size_t(Hyphen)); |
| 90 | result = CombineHashes(result, THash<size_t>()(TokenDelim)); |
| 91 | return result; |
| 92 | } |
| 93 | }; |
| 94 | |
| 95 | template <class T> |
| 96 | struct THash; |