Function parquet_statistics

datafusion/core/tests/sql/path_partition.rs:434–510

()

#[tokio::test]
async fn parquet_statistics() -> Result<()> {
    let mut config = SessionConfig::new();
    config.options_mut().execution.collect_statistics = true;
    let ctx = SessionContext::new_with_config(config);

    register_partitioned_alltypes_parquet(
        &ctx,
        &[
            "year=2021/month=09/day=09/file.parquet",
            "year=2021/month=10/day=09/file.parquet",
            "year=2021/month=10/day=28/file.parquet",
        ],
        &[
            ("year", DataType::Int32),
            ("month", DataType::Utf8),
            ("day", DataType::Utf8),
        ],
        "mirror:///",
        // This is the only file we found in the test set with
        // actual stats. It has 1 column / 1 row.
        "single_nan.parquet",
    )
    .await;

    //// NO PROJECTION ////
    let dataframe = ctx.sql("SELECT * FROM t").await?;
    let physical_plan = dataframe.create_physical_plan().await?;
    let schema = physical_plan.schema();
    assert_eq!(schema.fields().len(), 4);

    let stat_cols = physical_plan
        .partition_statistics(None)?
        .column_statistics
        .clone();
    assert_eq!(stat_cols.len(), 4);
    // stats for the first col are read from the parquet file
    assert_eq!(stat_cols[0].null_count, Precision::Exact(3));
    // Partition column statistics (year=2021 for all 3 rows)
    assert_eq!(stat_cols[1].null_count, Precision::Exact(0));
    assert_eq!(
        stat_cols[1].min_value,
        Precision::Exact(ScalarValue::Int32(Some(2021)))
    );
    assert_eq!(
        stat_cols[1].max_value,
        Precision::Exact(ScalarValue::Int32(Some(2021)))
    );
    // month and day are Utf8 partition columns with statistics
    assert_eq!(stat_cols[2].null_count, Precision::Exact(0));
    assert_eq!(stat_cols[3].null_count, Precision::Exact(0));

    //// WITH PROJECTION ////
    let dataframe = ctx.sql("SELECT mycol, day FROM t WHERE day='28'").await?;
    let physical_plan = dataframe.create_physical_plan().await?;
    let schema = physical_plan.schema();
    assert_eq!(schema.fields().len(), 2);

    let stat_cols = physical_plan