| 75 | } |
| 76 | |
| 77 | static bool HasChars(const TWideToken& token) { |
| 78 | TChecker checker; |
| 79 | const TTokenStructure& subtokens = token.SubTokens; |
| 80 | const wchar16* const s = token.Token; // source character sequence |
| 81 | size_t i = 0; // the index of the current source character |
| 82 | size_t t = 0; // the index of the next source subtoken |
| 83 | |
| 84 | while (i < token.Leng) { |
| 85 | if (t < subtokens.size()) { |
| 86 | if (i >= subtokens[t].Pos && i < subtokens[t].EndPos()) { |
| 87 | // inside a token |
| 88 | if (checker.Check(s[i])) { |
| 89 | return true; |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | ++i; |
| 94 | |
| 95 | if (i >= subtokens[t].EndPos()) { |
| 96 | ++t; |
| 97 | } |
| 98 | } else { |
| 99 | break; |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | return false; |
| 104 | } |
| 105 | |
| 106 | private: |
| 107 | const TWideToken& FilterTokenWithSubtokens(const TWideToken& token) { |