Method enableStringInterning

buffer.go:694–734

enableStringInterning analyzes columns and enables interning for low-cardinality string columns This can save 30-70% memory for datasets with repeated categorical values

()

// enableStringInterning analyzes columns and enables interning for low-cardinality string columns
// This can save 30-70% memory for datasets with repeated categorical values
func (b *Buffer) enableStringInterning() {
	b.mu.Lock()
	defer b.mu.Unlock()

	if b.rowLen < 100 {
		return // Too small to benefit
	}

	// Initialize interning structures
	b.interners = make([]*stringInterner, b.colLen)
	b.internCols = make([]bool, b.colLen)

	// Analyze each column
	for col := 0; col < b.colLen; col++ {
		// Skip non-string columns
		if b.colType[col] != colTypeStr {
			continue
		}

		// Get column data
		colData := make([]string, b.rowLen)
		for row := 0; row < b.rowLen; row++ {
			if col < len(b.cont[row]) {
				colData[row] = b.cont[row][col]
			}
		}

		// Check if column should be interned (low cardinality)
		if shouldInternColumn(colData, internCardinalityThreshold) {
			b.interners[col] = newStringInterner()
			b.internCols[col] = true

			// Intern existing values
			for row := 0; row < b.rowLen; row++ {
				if col < len(b.cont[row]) {
					b.cont[row][col] = b.interners[col].intern(b.cont[row][col])
				}
			}
		}
	}
}

// internValue interns a string value for a specific column if interning is enabled
func (b *Buffer) internValue(col int, value string) string {