Method Tokenize

pkg/sql/tokenizer/tokenizer.go:422–531

Tokenize converts raw SQL bytes into a slice of tokens with position information. This is the main entry point for tokenization. It performs zero-copy tokenization directly on the input byte slice and returns tokens with precise start/end positions. Performance: 8M+ tokens/sec sustained throughput

(input []byte)

//	tokens, _ := tkz.Tokenize([]byte(sql))
//	// Correctly tokenizes Unicode identifiers and string literals
func (t *Tokenizer) Tokenize(input []byte) ([]models.TokenWithSpan, error) {
	// Record start time for metrics
	startTime := time.Now()

	// Validate input size to prevent DoS attacks
	if len(input) > MaxInputSize {
		err := errors.InputTooLargeError(int64(len(input)), MaxInputSize, models.Location{Line: 1, Column: 0})
		metrics.RecordTokenization(time.Since(startTime), len(input), err)
		return nil, err
	}

	// Reset state
	t.Reset()
	t.input = input

	// Pre-allocate line starts slice - reuse if possible
	estimatedLines := len(input)/50 + 1 // Estimate 50 chars per line + 1 for initial 0
	if cap(t.lineStarts) < estimatedLines {
		t.lineStarts = make([]int, 0, estimatedLines)
	} else {
		t.lineStarts = t.lineStarts[:0]
	}
	t.lineStarts = append(t.lineStarts, 0)

	// Pre-scan input to build line start indices
	for i := 0; i < len(t.input); i++ {
		if t.input[i] == '\n' {
			t.lineStarts = append(t.lineStarts, i+1)
		}
	}

	// Pre-allocate token slice with better capacity estimation
	// More accurate estimation based on typical SQL token density
	estimatedTokens := len(input) / 4
	if estimatedTokens < 16 {
		estimatedTokens = 16 // At least 16 tokens
	}
	tokens := make([]models.TokenWithSpan, 0, estimatedTokens)

	// Get a buffer from the pool for string operations
	buf := getBuffer()
	defer putBuffer(buf)

	var tokenErr error
	func() {
		// Ensure proper cleanup even if we panic
		defer func() {
			if r := recover(); r != nil {
				tokenErr = errors.TokenizerPanicError(r, t.getCurrentPosition())
			}
		}()

		for t.pos.Index < len(t.input) {
			t.skipWhitespace()

			if t.pos.Index >= len(t.input) {
				break
			}