Function TestBufferStringInterning

string_interning_test.go:94–140

(t *testing.T)

}

func TestBufferStringInterning(t *testing.T) {
	// Create test data with repeated categorical values
	data := [][]string{
		{"Category", "Value"},
		{"A", "100"},
		{"B", "200"},
		{"A", "300"}, // Repeated
		{"C", "400"},
		{"A", "500"}, // Repeated
		{"B", "600"}, // Repeated
	}

	// Add more rows to make it worthwhile
	for i := 0; i < 100; i++ {
		data = append(data, []string{"A", fmt.Sprintf("%d", i)})
		data = append(data, []string{"B", fmt.Sprintf("%d", i+1000)})
		data = append(data, []string{"C", fmt.Sprintf("%d", i+2000)})
	}

	b, err := createNewBufferWithData(data, false)
	if err != nil {
		t.Fatalf("Failed to create buffer: %v", err)
	}

	b.rowFreeze = 1

	// Enable string interning
	b.enableStringInterning()

	// Check that interning is enabled
	stats := b.getInterningStats()
	if !stats["enabled"].(bool) {
		t.Error("String interning should be enabled")
	}

	// Category column (0) should be interned (low cardinality)
	if !b.internCols[0] {
		t.Error("Category column should be interned")
	}

	// Value column (1) should NOT be interned (high cardinality)
	if b.internCols[1] {
		t.Error("Value column should not be interned")
	}

	t.Logf("Interning stats: %+v", stats)
}

func TestStringInterningMemorySavings(t *testing.T) {
	// Create large dataset with repeated values