| 99 | } |
| 100 | |
| 101 | func TestTokenizer_SpecialOperators(t *testing.T) { |
| 102 | tests := []struct { |
| 103 | input string |
| 104 | expected []struct { |
| 105 | tokenType models.TokenType |
| 106 | value string |
| 107 | } |
| 108 | }{ |
| 109 | { |
| 110 | input: "x::text", |
| 111 | expected: []struct { |
| 112 | tokenType models.TokenType |
| 113 | value string |
| 114 | }{ |
| 115 | {models.TokenTypeIdentifier, "x"}, |
| 116 | {models.TokenTypeDoubleColon, "::"}, |
| 117 | {models.TokenTypeIdentifier, "text"}, |
| 118 | }, |
| 119 | }, |
| 120 | { |
| 121 | input: "data=>>'field'", |
| 122 | expected: []struct { |
| 123 | tokenType models.TokenType |
| 124 | value string |
| 125 | }{ |
| 126 | {models.TokenTypeIdentifier, "data"}, |
| 127 | {models.TokenTypeRArrow, "=>"}, |
| 128 | {models.TokenTypeGt, ">"}, |
| 129 | {models.TokenTypeSingleQuotedString, "field"}, |
| 130 | }, |
| 131 | }, |
| 132 | } |
| 133 | |
| 134 | for _, test := range tests { |
| 135 | tokenizer, err := New() |
| 136 | if err != nil { |
| 137 | t.Fatalf("New() error = %v", err) |
| 138 | } |
| 139 | tokens, err := tokenizer.Tokenize([]byte(test.input)) |
| 140 | if err != nil { |
| 141 | t.Fatalf("Tokenize() error = %v", err) |
| 142 | } |
| 143 | if len(tokens)-1 != len(test.expected) { // -1 for EOF |
| 144 | t.Logf("Tokens for %q:", test.input) |
| 145 | for i, token := range tokens { |
| 146 | if i < len(tokens)-1 { // Skip EOF |
| 147 | t.Logf("Token %d: Type=%v, Value=%q", i, token.Token.Type, token.Token.Value) |
| 148 | } |
| 149 | } |
| 150 | t.Fatalf("wrong number of tokens for %q, got %d, expected %d", test.input, len(tokens)-1, len(test.expected)) |
| 151 | } |
| 152 | for i, exp := range test.expected { |
| 153 | if tokens[i].Token.Type != exp.tokenType { |
| 154 | t.Errorf("wrong type for token %d in %q, got %v, expected %v", i, test.input, tokens[i].Token.Type, exp.tokenType) |
| 155 | } |
| 156 | if tokens[i].Token.Value != exp.value { |
| 157 | t.Errorf("wrong value for token %d in %q, got %v, expected %v", i, test.input, tokens[i].Token.Value, exp.value) |
| 158 | } |