| 96 | } |
| 97 | |
| 98 | void AddLastToken(const wchar16* tokstart, const wchar16* tokend) { |
| 99 | // - CurCharSpan.Len assigned to 0 in AddToken() because in case of multitoken with '.' at the end, for |
| 100 | // example: " well-formed. " parser already called to %add_token because '.' can be delimiter of the next token |
| 101 | if (CurCharSpan.Len) { |
| 102 | const wchar16* const actualStart = tokstart + CurCharSpan.Pos; |
| 103 | // for ex. "5% " can have (actualStart == tokend) because '%' could be part of the next token with utf8 characters |
| 104 | if (actualStart < tokend) { |
| 105 | const size_t actualLen = tokend - actualStart; |
| 106 | if (CurCharSpan.Len != actualLen) // for example "WORD% NEXTWORD" - '%' could be part of UTF8 encoded character and already counted... |
| 107 | CurCharSpan.Len = actualLen; |
| 108 | AddToken(); |
| 109 | } else |
| 110 | CancelToken(); |
| 111 | } else |
| 112 | CancelToken(); |
| 113 | |
| 114 | TTokenStructure& tokens = Multitoken.SubTokens; |
| 115 | if (!tokens.empty()) |
| 116 | tokens.back().TokenDelim = TOKDELIM_NULL; // reset delimiter if any |
| 117 | } |
| 118 | |
| 119 | //! correct the last token if it contains words and numbers and changes length of multitoken |
| 120 | //! @param len length of multitoken (including all subtokens), for ex. (te - ts) |