Function normalizeToken

pkg/sql/parser/preprocess.go:132–165

normalizeToken maps a TokenWithSpan to a canonical form expected by the parser. Type mismatches that existed in the tokenizer are corrected here.

(t models.TokenWithSpan)

// normalizeToken maps a TokenWithSpan to a canonical form expected by the
// parser. Type mismatches that existed in the tokenizer are corrected here.
func normalizeToken(t models.TokenWithSpan) models.TokenWithSpan {
	// TokenTypeMul (*) is the same symbol as TokenTypeAsterisk for the
	// parser. The parser explicitly checks both types wherever '*' can appear
	// (SELECT list wildcard, table.* qualified wildcard, COUNT(*) argument).
	// No remap is performed here to preserve the distinction for binary multiply.

	// Aggregate-function tokens (COUNT, SUM, AVG, MIN, MAX) must be treated
	// as identifiers so they can be used as function-call names.
	switch t.Token.Type {
	case models.TokenTypeCount, models.TokenTypeSum, models.TokenTypeAvg,
		models.TokenTypeMin, models.TokenTypeMax:
		t.Token.Type = models.TokenTypeIdentifier
		return t
	}

	// Identifier tokens whose *value* is a keyword that the parser dispatches
	// on by type need to be remapped.
	if t.Token.Type == models.TokenTypeIdentifier {
		if remapped := identifierKeywordType(t.Token.Value); remapped != models.TokenTypeUnknown {
			t.Token.Type = remapped
		}
		return t
	}

	// Generic TokenTypeKeyword tokens need a specific type for the parser.
	if t.Token.Type == models.TokenTypeKeyword {
		if remapped := keywordType(t.Token.Value); remapped != models.TokenTypeUnknown {
			t.Token.Type = remapped
		}
		return t
	}

	return t
}

// toUpper converts s to uppercase using a pooled byte buffer for short strings.
func toUpper(s string) string {