MCPcopy Create free account
hub / github.com/apache/datafusion / infer_options

Method infer_options

datafusion/core/src/datasource/listing/table.rs:50–101  ·  view source on GitHub ↗
(
        self,
        state: &dyn Session,
    )

Source from the content-addressed store, hash-verified

48#[async_trait]
49impl ListingTableConfigExt for ListingTableConfig {
50 async fn infer_options(
51 self,
52 state: &dyn Session,
53 ) -> datafusion_common::Result<ListingTableConfig> {
54 let store = if let Some(url) = self.table_paths.first() {
55 state.runtime_env().object_store(url)?
56 } else {
57 return Ok(self);
58 };
59
60 let file = self
61 .table_paths
62 .first()
63 .unwrap()
64 .list_all_files(state, store.as_ref(), "")
65 .await?
66 .next()
67 .await
68 .ok_or_else(|| internal_datafusion_err!("No files for table"))??;
69
70 let (file_extension, maybe_compression_type) =
71 ListingTableConfig::infer_file_extension_and_compression_type(
72 file.location.as_ref(),
73 )?;
74
75 let mut format_options = HashMap::new();
76 if let Some(ref compression_type) = maybe_compression_type {
77 format_options
78 .insert("format.compression".to_string(), compression_type.clone());
79 }
80 let state = state.as_any().downcast_ref::<SessionState>().unwrap();
81 let file_format = state
82 .get_file_format_factory(&file_extension)
83 .ok_or(config_datafusion_err!(
84 "No file_format found with extension {file_extension}"
85 ))?
86 .create(state, &format_options)?;
87
88 let listing_file_extension =
89 if let Some(compression_type) = maybe_compression_type {
90 format!("{}.{}", &file_extension, &compression_type)
91 } else {
92 file_extension
93 };
94
95 let listing_options = ListingOptions::new(file_format)
96 .with_file_extension(listing_file_extension)
97 .with_target_partitions(state.config().target_partitions())
98 .with_collect_stat(state.config().collect_statistics());
99
100 Ok(self.with_listing_options(listing_options))
101 }
102
103 async fn infer(self, state: &dyn Session) -> datafusion_common::Result<Self> {
104 self.infer_options(state).await?.infer_schema(state).await

Calls 15

newFunction · 0.85
list_all_filesMethod · 0.80
with_collect_statMethod · 0.80
with_file_extensionMethod · 0.80
target_partitionsMethod · 0.80
collect_statisticsMethod · 0.80
with_listing_optionsMethod · 0.80
firstMethod · 0.45
object_storeMethod · 0.45
runtime_envMethod · 0.45
nextMethod · 0.45