readPunctuation picks out punctuation or operator tokens
()
| 1263 | |
| 1264 | // readPunctuation picks out punctuation or operator tokens |
| 1265 | func (t *Tokenizer) readPunctuation() (models.Token, error) { |
| 1266 | if t.pos.Index >= len(t.input) { |
| 1267 | return models.Token{}, errors.IncompleteStatementError(t.getCurrentPosition(), string(t.input)) |
| 1268 | } |
| 1269 | r, size := utf8.DecodeRune(t.input[t.pos.Index:]) |
| 1270 | switch r { |
| 1271 | case '(': |
| 1272 | t.pos.AdvanceRune(r, size) |
| 1273 | return models.Token{Type: models.TokenTypeLeftParen, Value: "("}, nil |
| 1274 | case ')': |
| 1275 | t.pos.AdvanceRune(r, size) |
| 1276 | return models.Token{Type: models.TokenTypeRightParen, Value: ")"}, nil |
| 1277 | case '[': |
| 1278 | // SQL Server dialect: [identifier] is a quoted identifier |
| 1279 | if t.dialect == keywords.DialectSQLServer { |
| 1280 | t.pos.AdvanceRune(r, size) // Consume [ |
| 1281 | var ident []byte |
| 1282 | for t.pos.Index < len(t.input) { |
| 1283 | ch, chSize := utf8.DecodeRune(t.input[t.pos.Index:]) |
| 1284 | if ch == ']' { |
| 1285 | t.pos.AdvanceRune(ch, chSize) // Consume ] |
| 1286 | return models.Token{Type: models.TokenTypeIdentifier, Value: string(ident), Quote: '['}, nil |
| 1287 | } |
| 1288 | ident = append(ident, t.input[t.pos.Index:t.pos.Index+chSize]...) |
| 1289 | t.pos.AdvanceRune(ch, chSize) |
| 1290 | } |
| 1291 | return models.Token{}, errors.InvalidSyntaxError( |
| 1292 | "unterminated bracket identifier", |
| 1293 | t.getCurrentPosition(), |
| 1294 | string(t.input), |
| 1295 | ) |
| 1296 | } |
| 1297 | t.pos.AdvanceRune(r, size) |
| 1298 | return models.Token{Type: models.TokenTypeLBracket, Value: "["}, nil |
| 1299 | case ']': |
| 1300 | t.pos.AdvanceRune(r, size) |
| 1301 | return models.Token{Type: models.TokenTypeRBracket, Value: "]"}, nil |
| 1302 | case ',': |
| 1303 | t.pos.AdvanceRune(r, size) |
| 1304 | return models.Token{Type: models.TokenTypeComma, Value: ","}, nil |
| 1305 | case ';': |
| 1306 | t.pos.AdvanceRune(r, size) |
| 1307 | return models.Token{Type: models.TokenTypeSemicolon, Value: ";"}, nil |
| 1308 | case '.': |
| 1309 | t.pos.AdvanceRune(r, size) |
| 1310 | return models.Token{Type: models.TokenTypeDot, Value: "."}, nil |
| 1311 | case '+': |
| 1312 | t.pos.AdvanceRune(r, size) |
| 1313 | return models.Token{Type: models.TokenTypePlus, Value: "+"}, nil |
| 1314 | case '-': |
| 1315 | t.pos.AdvanceRune(r, size) |
| 1316 | if t.pos.Index < len(t.input) { |
| 1317 | nxtR, nxtSize := utf8.DecodeRune(t.input[t.pos.Index:]) |
| 1318 | if nxtR == '>' { |
| 1319 | t.pos.AdvanceRune(nxtR, nxtSize) |
| 1320 | // Check for ->> (JSON text extraction) |
| 1321 | if t.pos.Index < len(t.input) { |
| 1322 | thirdR, thirdSize := utf8.DecodeRune(t.input[t.pos.Index:]) |
no test coverage detected