Interface FileSource

datafusion/datasource/src/file.rs:65–354

File format specific behaviors for [`DataSource`] # Schema information There are two important schemas for a [`FileSource`]: 1. [`Self::table_schema`] -- the schema for the overall table (file data plus partition columns) 2. The logical output schema, comprised of [`Self::table_schema`] with [`Self::projection`] applied See more details on specific implementations: [`ArrowSource`](https://docs.r

///
/// [`DataSource`]: crate::source::DataSource
pub trait FileSource: Any + Send + Sync {
    /// Creates a `dyn FileOpener` based on given parameters.
    ///
    /// Note: File sources with a native morsel implementation should return an
    /// error from this method and implementing [`Self::create_morselizer`] instead.
    fn create_file_opener(
        &self,
        object_store: Arc<dyn ObjectStore>,
        base_config: &FileScanConfig,
        partition: usize,
    ) -> Result<Arc<dyn FileOpener>>;

    /// Creates a `dyn Morselizer` based on given parameters.
    ///
    /// The default implementation preserves existing behavior by adapting the
    /// legacy [`FileOpener`] API into a [`Morselizer`].
    ///
    /// It is preferred to implement the [`Morselizer`] API directly by
    /// implementing this method.
    fn create_morselizer(
        &self,
        object_store: Arc<dyn ObjectStore>,
        base_config: &FileScanConfig,
        partition: usize,
    ) -> Result<Box<dyn Morselizer>> {
        let opener = self.create_file_opener(object_store, base_config, partition)?;
        Ok(Box::new(FileOpenerMorselizer::new(opener)))
    }

    /// Returns the table schema for the overall table (including partition columns, if any)
    ///
    /// This method returns the unprojected schema: the full schema of the data
    /// without [`Self::projection`] applied.
    ///
    /// The output schema of this `FileSource` is this TableSchema
    /// with [`Self::projection`] applied.
    ///
    /// Use [`ProjectionExprs::project_schema`] to get the projected schema
    /// after applying the projection.
    fn table_schema(&self) -> &crate::table_schema::TableSchema;

    /// Initialize new type with batch size configuration
    fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>;

    /// Returns the filter expression that will be applied *during* the file scan.
    ///
    /// These expressions are in terms of the unprojected [`Self::table_schema`].
    fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {
        None
    }

    /// Return the projection that will be applied to the output stream on top
    /// of [`Self::table_schema`].
    ///
    /// Note you can use [`ProjectionExprs::project_schema`] on the table
    /// schema to get the effective output schema of this source.
    fn projection(&self) -> Option<&ProjectionExprs> {
        None