| 404 | } |
| 405 | |
| 406 | void TNlpParser::ProcessSurrogatePairs(const wchar16* ts, const wchar16* te) { |
| 407 | const wchar16 brokenRune = BROKEN_RUNE; |
| 408 | const wchar16* lead = nullptr; |
| 409 | for (const wchar16* p = ts; p != te; ++p) { |
| 410 | if (IsW16SurrogateLead(*p)) { |
| 411 | if (lead) |
| 412 | MakeEntry(&brokenRune, 1, NLP_MISCTEXT); |
| 413 | lead = p; |
| 414 | } else if (IsW16SurrogateTail(*p)) { |
| 415 | if (lead) { |
| 416 | Base.AddIdeograph(2); |
| 417 | Y_ASSERT(Base.GetTokenCount() == 1); |
| 418 | MakeMultitokenEntry(Base.GetToken(0), lead); |
| 419 | Base.ResetTokens(); |
| 420 | } else |
| 421 | MakeEntry(&brokenRune, 1, NLP_MISCTEXT); |
| 422 | lead = nullptr; |
| 423 | } else |
| 424 | Y_ASSERT(!"invalid character"); |
| 425 | } |
| 426 | if (lead) |
| 427 | MakeEntry(&brokenRune, 1, NLP_MISCTEXT); |
| 428 | } |
| 429 | |
| 430 | void TNlpParser::ProcessIdeographs(const wchar16* ts, const wchar16* te) { |
| 431 | for (const wchar16* p = ts; p != te; ++p) { |
nothing calls this directly
no test coverage detected