Method nextToken

pkg/sql/tokenizer/tokenizer.go:705–728

nextToken picks out the next token from the input

()

// nextToken picks out the next token from the input
func (t *Tokenizer) nextToken() (models.Token, error) {
	if t.pos.Index >= len(t.input) {
		return models.Token{Type: models.TokenTypeEOF}, nil
	}

	// Fast path for common cases
	r, _ := utf8.DecodeRune(t.input[t.pos.Index:])
	switch {
	case isIdentifierStart(r):
		return t.readIdentifier()
	case r >= '0' && r <= '9':
		return t.readNumber(nil)
	case r == '"' || isUnicodeQuote(r):
		return t.readQuotedIdentifier()
	case r == '`':
		// MySQL-style backtick identifier
		return t.readBacktickIdentifier()
	case r == '\'' || r == '\u2018' || r == '\u2019' || r == '\u00AB' || r == '\u00BB':
		return t.readQuotedString(r)
	}

	// Slower path for punctuation and operators
	return t.readPunctuation()
}

// isIdentifierStart checks if a rune can start an identifier
func isIdentifierStart(r rune) bool {