Method TokenizeContext

pkg/sql/tokenizer/tokenizer.go:550–671

TokenizeContext processes the input and returns tokens with context support for cancellation. It checks the context at regular intervals (every 100 tokens) to enable fast cancellation. Returns context.Canceled or context.DeadlineExceeded when the context is cancelled. This method is useful for: - L

(ctx context.Context, input []byte)

//	    // Handle timeout
//	}
func (t *Tokenizer) TokenizeContext(ctx context.Context, input []byte) ([]models.TokenWithSpan, error) {
	// Check context before starting
	if err := ctx.Err(); err != nil {
		return nil, err
	}

	// Record start time for metrics
	startTime := time.Now()

	// Validate input size to prevent DoS attacks
	if len(input) > MaxInputSize {
		err := errors.InputTooLargeError(int64(len(input)), MaxInputSize, models.Location{Line: 1, Column: 0})
		metrics.RecordTokenization(time.Since(startTime), len(input), err)
		return nil, err
	}

	// Reset state
	t.Reset()
	t.input = input

	// Pre-allocate line starts slice - reuse if possible
	estimatedLines := len(input)/50 + 1 // Estimate 50 chars per line + 1 for initial 0
	if cap(t.lineStarts) < estimatedLines {
		t.lineStarts = make([]int, 0, estimatedLines)
	} else {
		t.lineStarts = t.lineStarts[:0]
	}
	t.lineStarts = append(t.lineStarts, 0)

	// Pre-scan input to build line start indices
	for i := 0; i < len(t.input); i++ {
		if t.input[i] == '\n' {
			t.lineStarts = append(t.lineStarts, i+1)
		}
	}

	// Pre-allocate token slice with better capacity estimation
	estimatedTokens := len(input) / 4
	if estimatedTokens < 16 {
		estimatedTokens = 16 // At least 16 tokens
	}
	tokens := make([]models.TokenWithSpan, 0, estimatedTokens)

	// Get a buffer from the pool for string operations
	buf := getBuffer()
	defer putBuffer(buf)

	var tokenErr error
	func() {
		// Ensure proper cleanup even if we panic
		defer func() {
			if r := recover(); r != nil {
				tokenErr = errors.TokenizerPanicError(r, t.getCurrentPosition())
			}
		}()

		for t.pos.Index < len(t.input) {
			// Check context every 100 tokens for cancellation