Return a new `DataFrame` that has statistics for a DataFrame. Only summarizes numeric datatypes at the moment and returns nulls for non numeric datatypes. The output format is modeled after pandas # Example ``` # use datafusion::prelude::*; # use datafusion::error::Result; # use arrow::util::pretty; # use datafusion_common::assert_batches_sorted_eq; # #[tokio::main] # async fn main() -> Result<(
(self)
| 1011 | /// # } |
| 1012 | /// ``` |
| 1013 | pub async fn describe(self) -> Result<Self> { |
| 1014 | //the functions now supported |
| 1015 | let supported_describe_functions = |
| 1016 | vec!["count", "null_count", "mean", "std", "min", "max", "median"]; |
| 1017 | |
| 1018 | let original_schema_fields = self.schema().fields().iter(); |
| 1019 | |
| 1020 | //define describe column |
| 1021 | let mut describe_schemas = vec![Field::new("describe", DataType::Utf8, false)]; |
| 1022 | describe_schemas.extend(original_schema_fields.clone().map(|field| { |
| 1023 | if field.data_type().is_numeric() { |
| 1024 | Field::new(field.name(), DataType::Float64, true) |
| 1025 | } else { |
| 1026 | Field::new(field.name(), DataType::Utf8, true) |
| 1027 | } |
| 1028 | })); |
| 1029 | |
| 1030 | //collect recordBatch |
| 1031 | let describe_record_batch = [ |
| 1032 | // count aggregation |
| 1033 | self.clone().aggregate( |
| 1034 | vec![], |
| 1035 | original_schema_fields |
| 1036 | .clone() |
| 1037 | .map(|f| count(ident(f.name())).alias(f.name())) |
| 1038 | .collect::<Vec<_>>(), |
| 1039 | ), |
| 1040 | // null_count aggregation |
| 1041 | self.clone().aggregate( |
| 1042 | vec![], |
| 1043 | original_schema_fields |
| 1044 | .clone() |
| 1045 | .map(|f| { |
| 1046 | sum(case(is_null(ident(f.name()))) |
| 1047 | .when(lit(true), lit(1)) |
| 1048 | .otherwise(lit(0)) |
| 1049 | .unwrap()) |
| 1050 | .alias(f.name()) |
| 1051 | }) |
| 1052 | .collect::<Vec<_>>(), |
| 1053 | ), |
| 1054 | // mean aggregation |
| 1055 | self.clone().aggregate( |
| 1056 | vec![], |
| 1057 | original_schema_fields |
| 1058 | .clone() |
| 1059 | .filter(|f| f.data_type().is_numeric()) |
| 1060 | .map(|f| avg(ident(f.name())).alias(f.name())) |
| 1061 | .collect::<Vec<_>>(), |
| 1062 | ), |
| 1063 | // std aggregation |
| 1064 | self.clone().aggregate( |
| 1065 | vec![], |
| 1066 | original_schema_fields |
| 1067 | .clone() |
| 1068 | .filter(|f| f.data_type().is_numeric()) |
| 1069 | .map(|f| stddev(ident(f.name())).alias(f.name())) |
| 1070 | .collect::<Vec<_>>(), |