Generates test files with min-max statistics in different overlap patterns. Used by tests and benchmarks. # Overlap Factors The `overlap_factor` parameter controls how much the value ranges in generated test files overlap: - `0.0`: No overlap between files (completely disjoint ranges) - `0.2`: Low overlap (20% of the range size overlaps with adjacent files) - `0.5`: Medium overlap (50% of range
(num_files: usize, overlap_factor: f64)
| 540 | /// File 3: [60, 160] |
| 541 | /// File 4: [80, 180] |
| 542 | pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGroup> { |
| 543 | let mut files = Vec::with_capacity(num_files); |
| 544 | if num_files == 0 { |
| 545 | return vec![]; |
| 546 | } |
| 547 | let range_size = if overlap_factor == 0.0 { |
| 548 | 100 / num_files as i64 |
| 549 | } else { |
| 550 | (100.0 / (overlap_factor * num_files as f64)).max(1.0) as i64 |
| 551 | }; |
| 552 | |
| 553 | for i in 0..num_files { |
| 554 | let base = (i as f64 * range_size as f64 * (1.0 - overlap_factor)) as i64; |
| 555 | let min = base as f64; |
| 556 | let max = (base + range_size) as f64; |
| 557 | |
| 558 | let file = PartitionedFile { |
| 559 | object_meta: ObjectMeta { |
| 560 | location: Path::from(format!("file_{i}.parquet")), |
| 561 | last_modified: chrono::Utc::now(), |
| 562 | size: 1000, |
| 563 | e_tag: None, |
| 564 | version: None, |
| 565 | }, |
| 566 | partition_values: vec![], |
| 567 | range: None, |
| 568 | statistics: Some(Arc::new(Statistics { |
| 569 | num_rows: Precision::Exact(100), |
| 570 | total_byte_size: Precision::Exact(1000), |
| 571 | column_statistics: vec![ColumnStatistics { |
| 572 | null_count: Precision::Exact(0), |
| 573 | max_value: Precision::Exact(ScalarValue::Float64(Some(max))), |
| 574 | min_value: Precision::Exact(ScalarValue::Float64(Some(min))), |
| 575 | sum_value: Precision::Absent, |
| 576 | distinct_count: Precision::Absent, |
| 577 | byte_size: Precision::Absent, |
| 578 | }], |
| 579 | })), |
| 580 | ordering: None, |
| 581 | extensions: FileExtensions::new(), |
| 582 | metadata_size_hint: None, |
| 583 | table_reference: None, |
| 584 | }; |
| 585 | files.push(file); |
| 586 | } |
| 587 | |
| 588 | vec![FileGroup::new(files)] |
| 589 | } |
| 590 | |
| 591 | // Helper function to verify that files within each group maintain sort order |
| 592 | /// Used by tests and benchmarks |
searching dependent graphs…