Function Tokens

postgres/parser/parser/scan.go:1037–1051

Tokens decomposes the input into lexical tokens.

(sql string)

// Tokens decomposes the input into lexical tokens.
func Tokens(sql string) (tokens []TokenString, ok bool) {
	s := makeScanner(sql)
	for {
		var lval sqlSymType
		s.scan(&lval)
		if lval.id == ERROR {
			return nil, false
		}
		if lval.id == 0 {
			break
		}
		tokens = append(tokens, TokenString{TokenID: lval.id, Str: lval.str})
	}
	return tokens, true
}

// TokenString is the unit value returned by Tokens.
type TokenString struct {