Method read_to_delimited_chunks

datafusion/datasource-csv/src/file_format.rs:143–169

Return a newline delimited stream from the specified file on Stream, decompressing if necessary Each returned `Bytes` has a whole number of newline delimited rows

(
        &self,
        store: &Arc<dyn ObjectStore>,
        object: &ObjectMeta,
    )

    /// Stream, decompressing if necessary
    /// Each returned `Bytes` has a whole number of newline delimited rows
    async fn read_to_delimited_chunks<'a>(
        &self,
        store: &Arc<dyn ObjectStore>,
        object: &ObjectMeta,
    ) -> BoxStream<'a, Result<Bytes>> {
        // stream to only read as many rows as needed into memory
        let stream = store
            .get(&object.location)
            .await
            .map_err(|e| DataFusionError::ObjectStore(Box::new(e)));
        let stream = match stream {
            Ok(stream) => self
                .read_to_delimited_chunks_from_stream(
                    stream
                        .into_stream()
                        .map_err(|e| DataFusionError::ObjectStore(Box::new(e)))
                        .boxed(),
                )
                .await
                .map_err(DataFusionError::from)
                .left_stream(),
            Err(e) => {
                futures::stream::once(futures::future::ready(Err(e))).right_stream()
            }
        };
        stream.boxed()
    }

    /// Convert a stream of bytes into a stream of of [`Bytes`] containing newline
    /// delimited CSV records, while accounting for `\` and `"`.