| 188 | Current->AddIdeograph(len); |
| 189 | } |
| 190 | void AddLastToken(const wchar16* tokstart, const wchar16* tokend) { |
| 191 | // - CurCharSpan.Len assigned to 0 in AddToken() because in case of multitoken with '.' at the end, for |
| 192 | // example: " well-formed. " parser already called to %add_token because '.' can be delimiter of the next token |
| 193 | if (CurCharSpan.Len) { |
| 194 | const wchar16* const actualStart = tokstart + CurCharSpan.Pos; |
| 195 | // for ex. "5% " can have (actualStart == tokend) because '%' could be part of the next token with utf8 characters |
| 196 | if (actualStart < tokend) { |
| 197 | const size_t actualLen = tokend - actualStart; |
| 198 | if (CurCharSpan.Len != actualLen) // for example "WORD% NEXTWORD" - '%' could be part of UTF8 encoded character and already counted... |
| 199 | CurCharSpan.Len = actualLen; |
| 200 | AddToken(); |
| 201 | } else |
| 202 | CancelToken(); |
| 203 | } else |
| 204 | CancelToken(); |
| 205 | |
| 206 | if (Current->GetSubtokenCount()) |
| 207 | Current->SetTokenDelim(TOKDELIM_NULL); // reset delimiter if any |
| 208 | } |
| 209 | void UpdatePrefix(wchar16 c) { |
| 210 | Y_ASSERT(c == '#' || c == '@' || c == '$'); |
| 211 | PrefixLen = 1; // length of prefix can't be more than 1 |
no test coverage detected