MCPcopy Create free account
hub / github.com/codechenx/FastTableViewer / detectAllColumnTypes

Method detectAllColumnTypes

buffer.go:673–690  ·  view source on GitHub ↗

detectAllColumnTypes automatically detects types for all columns in parallel

()

Source from the content-addressed store, hash-verified

671
672// detectAllColumnTypes automatically detects types for all columns in parallel
673func (b *Buffer) detectAllColumnTypes() {
674 types := make([]int, b.colLen)
675 var wg sync.WaitGroup
676
677 for i := 0; i < b.colLen; i++ {
678 wg.Add(1)
679 go func(col int) {
680 defer wg.Done()
681 types[col] = b.autoDetectColumnType(col)
682 }(i)
683 }
684
685 wg.Wait()
686
687 for i, t := range types {
688 b.setColType(i, t)
689 }
690}
691
692// enableStringInterning analyzes columns and enables interning for low-cardinality string columns
693// This can save 30-70% memory for datasets with repeated categorical values

Callers 7

TestDetectAllColumnTypesFunction · 0.80
TestMultiColumnFilterFunction · 0.80
loadFileToBufferAsyncFunction · 0.80
loadFileToBufferFunction · 0.80
loadPipeToBufferAsyncFunction · 0.80
loadPipeToBufferFunction · 0.80

Calls 2

autoDetectColumnTypeMethod · 0.95
setColTypeMethod · 0.95

Tested by 3

TestDetectAllColumnTypesFunction · 0.64
TestMultiColumnFilterFunction · 0.64