MCPcopy Create free account
hub / github.com/apache/datafusion / create_morselizer

Method create_morselizer

datafusion/datasource-parquet/src/source.rs:542–617  ·  view source on GitHub ↗
(
        &self,
        object_store: Arc<dyn ObjectStore>,
        base_config: &FileScanConfig,
        partition: usize,
    )

Source from the content-addressed store, hash-verified

540 }
541
542 fn create_morselizer(
543 &self,
544 object_store: Arc<dyn ObjectStore>,
545 base_config: &FileScanConfig,
546 partition: usize,
547 ) -> datafusion_common::Result<Box<dyn Morselizer>> {
548 let expr_adapter_factory = base_config
549 .expr_adapter_factory
550 .clone()
551 .unwrap_or_else(|| Arc::new(DefaultPhysicalExprAdapterFactory) as _);
552
553 let parquet_file_reader_factory =
554 self.parquet_file_reader_factory.clone().unwrap_or_else(|| {
555 Arc::new(DefaultParquetFileReaderFactory::new(object_store)) as _
556 });
557
558 #[cfg(feature = "parquet_encryption")]
559 let file_decryption_properties = self
560 .table_parquet_options()
561 .crypto
562 .file_decryption
563 .clone()
564 .map(FileDecryptionProperties::try_from)
565 .transpose()?
566 .map(Arc::new);
567
568 let coerce_int96 = self
569 .table_parquet_options
570 .global
571 .coerce_int96
572 .as_ref()
573 .map(|time_unit| parse_coerce_int96_string(time_unit.as_str()).unwrap());
574 let coerce_int96_tz = self
575 .table_parquet_options
576 .global
577 .coerce_int96_tz
578 .as_ref()
579 .map(|tz| parse_coerce_int96_tz_string(tz))
580 .transpose()?;
581 if coerce_int96_tz.is_some() && coerce_int96.is_none() {
582 warn!(
583 "coerce_int96_tz is set but coerce_int96 is not; the timezone will be ignored"
584 );
585 }
586
587 Ok(Box::new(ParquetMorselizer {
588 partition_index: partition,
589 projection: self.projection.clone(),
590 batch_size: self
591 .batch_size
592 .expect("Batch size must set before creating ParquetMorselizer"),
593 limit: base_config.limit,
594 preserve_order: base_config.preserve_order,
595 predicate: self.predicate.clone(),
596 table_schema: self.table_schema.clone(),
597 metadata_size_hint: self.metadata_size_hint,
598 metrics: self.metrics().clone(),
599 parquet_file_reader_factory,

Callers

nothing calls this directly

Calls 15

newFunction · 0.85
table_parquet_optionsMethod · 0.80
is_noneMethod · 0.80
pushdown_filtersMethod · 0.80
reorder_filtersMethod · 0.80
enable_page_indexMethod · 0.80
bloom_filter_on_readMethod · 0.80

Tested by

no test coverage detected