normalizeIdent takes in a function that determines if a character is a legal identifier character, and a boolean toLower that indicates whether to set the identifier to lowercase when normalizing.
(lval ScanSymType, isIdentMiddle func(int) bool, toLower bool)
| 610 | // identifier character, and a boolean toLower that indicates whether to set the |
| 611 | // identifier to lowercase when normalizing. |
| 612 | func (s *Scanner) normalizeIdent(lval ScanSymType, isIdentMiddle func(int) bool, toLower bool) { |
| 613 | s.lastAttemptedID = int32(lexbase.IDENT) |
| 614 | s.pos-- |
| 615 | start := s.pos |
| 616 | isASCII := true |
| 617 | isLower := true |
| 618 | |
| 619 | // Consume the Scanner character by character, stopping after the last legal |
| 620 | // identifier character. By the end of this function, we need to |
| 621 | // lowercase and unicode normalize this identifier, which is expensive if |
| 622 | // there are actual unicode characters in it. If not, it's quite cheap - and |
| 623 | // if it's lowercase already, there's no work to do. Therefore, we keep track |
| 624 | // of whether the string is only ASCII or only ASCII lowercase for later. |
| 625 | for { |
| 626 | ch := s.peek() |
| 627 | if ch >= utf8.RuneSelf { |
| 628 | isASCII = false |
| 629 | } else if ch >= 'A' && ch <= 'Z' { |
| 630 | isLower = false |
| 631 | } |
| 632 | |
| 633 | if !isIdentMiddle(ch) { |
| 634 | break |
| 635 | } |
| 636 | |
| 637 | s.pos++ |
| 638 | } |
| 639 | |
| 640 | if toLower && !isLower && isASCII { |
| 641 | // We know that the identifier we've seen so far is ASCII, so we don't |
| 642 | // to unicode normalize. Instead, just lowercase as normal. |
| 643 | b := s.allocBytes(s.pos - start) |
| 644 | _ = b[s.pos-start-1] // For bounds check elimination. |
| 645 | for i, c := range s.in[start:s.pos] { |
| 646 | if c >= 'A' && c <= 'Z' { |
| 647 | c += 'a' - 'A' |
| 648 | } |
| 649 | b[i] = byte(c) |
| 650 | } |
| 651 | lval.SetStr(*(*string)(unsafe.Pointer(&b))) |
| 652 | } else if toLower && !isASCII { |
| 653 | // The string has unicode in it. No choice but to normalize and lowercase. |
| 654 | lval.SetStr(lexbase.NormalizeName(s.in[start:s.pos])) |
| 655 | } else if !toLower && !isASCII { |
| 656 | // The string has unicode in it. No choice but to normalize. |
| 657 | lval.SetStr(lexbase.NormalizeString(s.in[start:s.pos])) |
| 658 | } else { |
| 659 | // Don't do anything. |
| 660 | lval.SetStr(s.in[start:s.pos]) |
| 661 | } |
| 662 | } |
| 663 | |
| 664 | func (s *Scanner) scanIdent(lval ScanSymType) { |
| 665 | s.normalizeIdent(lval, lexbase.IsIdentMiddle, true /* toLower */) |
no test coverage detected