Compute partition stats (min/max/null_counts) across all entries.
(&self, entries: &[ManifestEntry])
| 870 | |
| 871 | /// Compute partition stats (min/max/null_counts) across all entries. |
| 872 | fn compute_partition_stats(&self, entries: &[ManifestEntry]) -> Result<BinaryTableStats> { |
| 873 | let partition_fields = self.table.schema().partition_fields(); |
| 874 | let num_fields = partition_fields.len(); |
| 875 | |
| 876 | if num_fields == 0 || entries.is_empty() { |
| 877 | return Ok(BinaryTableStats::new(vec![], vec![], vec![])); |
| 878 | } |
| 879 | |
| 880 | let data_types: Vec<_> = partition_fields |
| 881 | .iter() |
| 882 | .map(|f| f.data_type().clone()) |
| 883 | .collect(); |
| 884 | let mut mins: Vec<Option<Datum>> = vec![None; num_fields]; |
| 885 | let mut maxs: Vec<Option<Datum>> = vec![None; num_fields]; |
| 886 | let mut null_counts: Vec<i64> = vec![0; num_fields]; |
| 887 | |
| 888 | for entry in entries { |
| 889 | let partition_bytes = entry.partition(); |
| 890 | if partition_bytes.is_empty() { |
| 891 | continue; |
| 892 | } |
| 893 | let row = BinaryRow::from_serialized_bytes(partition_bytes)?; |
| 894 | for i in 0..num_fields { |
| 895 | match extract_datum(&row, i, &data_types[i])? { |
| 896 | Some(datum) => { |
| 897 | mins[i] = Some(match mins[i].take() { |
| 898 | Some(cur) if cur <= datum => cur, |
| 899 | Some(_) => datum.clone(), |
| 900 | None => datum.clone(), |
| 901 | }); |
| 902 | maxs[i] = Some(match maxs[i].take() { |
| 903 | Some(cur) if cur >= datum => cur, |
| 904 | Some(_) => datum, |
| 905 | None => datum, |
| 906 | }); |
| 907 | } |
| 908 | None => { |
| 909 | null_counts[i] += 1; |
| 910 | } |
| 911 | } |
| 912 | } |
| 913 | } |
| 914 | |
| 915 | let min_datums: Vec<_> = mins.iter().zip(data_types.iter()).collect(); |
| 916 | let max_datums: Vec<_> = maxs.iter().zip(data_types.iter()).collect(); |
| 917 | |
| 918 | let min_bytes = datums_to_binary_row(&min_datums); |
| 919 | let max_bytes = datums_to_binary_row(&max_datums); |
| 920 | let null_counts = null_counts.into_iter().map(Some).collect(); |
| 921 | |
| 922 | Ok(BinaryTableStats::new(min_bytes, max_bytes, null_counts)) |
| 923 | } |
| 924 | |
| 925 | /// Generate per-partition statistics from commit entries. |
| 926 | /// |
no test coverage detected