shouldInternColumn determines if a column should use string interning based on its cardinality (ratio of unique values to total values)
(values []string, threshold float64)
| 40 | // shouldInternColumn determines if a column should use string interning |
| 41 | // based on its cardinality (ratio of unique values to total values) |
| 42 | func shouldInternColumn(values []string, threshold float64) bool { |
| 43 | if len(values) < 100 { |
| 44 | return false // Too small to benefit |
| 45 | } |
| 46 | |
| 47 | // Sample the column to estimate cardinality |
| 48 | sampleSize := 1000 |
| 49 | if len(values) < sampleSize { |
| 50 | sampleSize = len(values) |
| 51 | } |
| 52 | |
| 53 | seen := make(map[string]bool, sampleSize) |
| 54 | for i := 0; i < sampleSize; i++ { |
| 55 | seen[values[i]] = true |
| 56 | } |
| 57 | |
| 58 | cardinality := float64(len(seen)) / float64(sampleSize) |
| 59 | return cardinality < threshold // Low cardinality = good for interning |
| 60 | } |
| 61 | |
| 62 | // Buffer represents a table data structure with concurrent access support |
| 63 | type Buffer struct { |
no outgoing calls