Detect the optimal codec for an i64 column by analyzing the data. For partitions ≥ CASCADE_THRESHOLD values, selects cascading codecs. For smaller partitions, falls back to legacy single-step codecs.
(values: &[i64])
| 49 | /// For partitions ≥ CASCADE_THRESHOLD values, selects cascading codecs. |
| 50 | /// For smaller partitions, falls back to legacy single-step codecs. |
| 51 | pub fn detect_i64_codec(values: &[i64]) -> ColumnCodec { |
| 52 | if values.len() < 2 { |
| 53 | return ColumnCodec::Delta; |
| 54 | } |
| 55 | |
| 56 | // Large partitions → cascading codec (FastLanes handles all patterns). |
| 57 | if values.len() >= CASCADE_THRESHOLD { |
| 58 | return ColumnCodec::DeltaFastLanesLz4; |
| 59 | } |
| 60 | |
| 61 | // Small partitions → legacy codecs. Analyze data to pick best one. |
| 62 | let sample_end = values.len().min(CODEC_SAMPLE_SIZE); |
| 63 | let sample = &values[..sample_end]; |
| 64 | |
| 65 | let mut zero_dod_count = 0usize; |
| 66 | let mut prev_delta: Option<i64> = None; |
| 67 | |
| 68 | for i in 1..sample.len() { |
| 69 | let delta = sample[i] - sample[i - 1]; |
| 70 | if let Some(pd) = prev_delta |
| 71 | && delta == pd |
| 72 | { |
| 73 | zero_dod_count += 1; |
| 74 | } |
| 75 | prev_delta = Some(delta); |
| 76 | } |
| 77 | |
| 78 | let total_deltas = sample.len() - 1; |
| 79 | let constant_rate_ratio = zero_dod_count as f64 / total_deltas.max(1) as f64; |
| 80 | |
| 81 | if constant_rate_ratio > 0.8 { |
| 82 | ColumnCodec::DoubleDelta |
| 83 | } else { |
| 84 | ColumnCodec::Delta |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | /// Detect the optimal codec for an f64 column by analyzing the data. |
| 89 | /// |