detectAllColumnTypes automatically detects types for all columns in parallel
()
| 671 | |
| 672 | // detectAllColumnTypes automatically detects types for all columns in parallel |
| 673 | func (b *Buffer) detectAllColumnTypes() { |
| 674 | types := make([]int, b.colLen) |
| 675 | var wg sync.WaitGroup |
| 676 | |
| 677 | for i := 0; i < b.colLen; i++ { |
| 678 | wg.Add(1) |
| 679 | go func(col int) { |
| 680 | defer wg.Done() |
| 681 | types[col] = b.autoDetectColumnType(col) |
| 682 | }(i) |
| 683 | } |
| 684 | |
| 685 | wg.Wait() |
| 686 | |
| 687 | for i, t := range types { |
| 688 | b.setColType(i, t) |
| 689 | } |
| 690 | } |
| 691 | |
| 692 | // enableStringInterning analyzes columns and enables interning for low-cardinality string columns |
| 693 | // This can save 30-70% memory for datasets with repeated categorical values |