readIdentifier reads an identifier (e.g. foo or foo.bar)
()
| 734 | |
| 735 | // readIdentifier reads an identifier (e.g. foo or foo.bar) |
| 736 | func (t *Tokenizer) readIdentifier() (models.Token, error) { |
| 737 | start := t.pos.Index |
| 738 | r, size := utf8.DecodeRune(t.input[t.pos.Index:]) |
| 739 | t.pos.AdvanceRune(r, size) |
| 740 | |
| 741 | // Read until we hit a non-identifier character |
| 742 | for t.pos.Index < len(t.input) { |
| 743 | r, size = utf8.DecodeRune(t.input[t.pos.Index:]) |
| 744 | if !isIdentifierChar(r) { |
| 745 | _ = size // Mark as intentionally unused |
| 746 | break |
| 747 | } |
| 748 | t.pos.AdvanceRune(r, size) |
| 749 | } |
| 750 | |
| 751 | ident := string(t.input[start:t.pos.Index]) |
| 752 | |
| 753 | // SQL Server dialect: N'...' national string literal |
| 754 | if t.dialect == keywords.DialectSQLServer && (ident == "N" || ident == "n") { |
| 755 | if t.pos.Index < len(t.input) && t.input[t.pos.Index] == '\'' { |
| 756 | tok, err := t.readQuotedString('\'') |
| 757 | if err != nil { |
| 758 | return models.Token{}, err |
| 759 | } |
| 760 | return models.Token{ |
| 761 | Type: models.TokenTypeNationalStringLiteral, |
| 762 | Value: tok.Value, |
| 763 | Quote: 'N', |
| 764 | }, nil |
| 765 | } |
| 766 | } |
| 767 | |
| 768 | word := &models.Word{ |
| 769 | Value: ident, |
| 770 | } |
| 771 | |
| 772 | // Determine token type based on whether it's a keyword |
| 773 | upperIdent := strings.ToUpper(ident) |
| 774 | tokenType, isKeyword := keywordTokenTypes[upperIdent] |
| 775 | if !isKeyword { |
| 776 | tokenType = models.TokenTypeIdentifier |
| 777 | } |
| 778 | |
| 779 | // Check if this could be the start of a compound keyword |
| 780 | if isCompoundKeywordStart(upperIdent) { |
| 781 | // Save current position |
| 782 | savePos := t.pos.Clone() |
| 783 | |
| 784 | // Skip whitespace |
| 785 | t.skipWhitespace() |
| 786 | |
| 787 | if t.pos.Index < len(t.input) { |
| 788 | // Try to read the next word |
| 789 | nextStart := t.pos.Index |
| 790 | r, size := utf8.DecodeRune(t.input[t.pos.Index:]) |
| 791 | if isIdentifierStart(r) { |
| 792 | t.pos.AdvanceRune(r, size) |
| 793 |
no test coverage detected