Function TestTokenizeContext_BasicSuccess

pkg/sql/tokenizer/context_test.go:25–65

TestTokenizeContext_BasicSuccess verifies that TokenizeContext works for valid SQL

(t *testing.T)

// TestTokenizeContext_BasicSuccess verifies that TokenizeContext works for valid SQL
func TestTokenizeContext_BasicSuccess(t *testing.T) {
	tests := []struct {
		name  string
		input string
	}{
		{
			name:  "simple select",
			input: "SELECT * FROM users",
		},
		{
			name:  "complex query with joins",
			input: "SELECT u.name, o.total FROM users u JOIN orders o ON u.id = o.user_id WHERE u.active = true",
		},
		{
			name:  "insert statement",
			input: "INSERT INTO users (name, email) VALUES ('John', 'john@example.com')",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			tkz := GetTokenizer()
			defer PutTokenizer(tkz)

			tokens, err := tkz.TokenizeContext(ctx, []byte(tt.input))
			if err != nil {
				t.Fatalf("TokenizeContext() error = %v", err)
			}

			if len(tokens) == 0 {
				t.Error("Expected tokens but got none")
			}

			// Verify last token is EOF (TokenTypeEOF = 0)
			if tokens[len(tokens)-1].Token.Type != 0 {
				t.Errorf("Expected last token to be EOF (0), got %d", tokens[len(tokens)-1].Token.Type)
			}
		})
	}
}

// TestTokenizeContext_CancelledContext verifies that cancellation is detected
func TestTokenizeContext_CancelledContext(t *testing.T) {