Function find_first_newline

datafusion/datasource/src/mod.rs:474–501

Asynchronously finds the position of the first newline character in a specified byte range within an object, such as a file, in an object store. This function scans the contents of the object starting from the specified `start` position up to the `end` position, looking for the first occurrence of a newline character. It returns the position of the first newline relative to the start of the range

(
    object_store: &Arc<dyn ObjectStore>,
    location: &Path,
    start: u64,
    end: u64,
    newline: u8,
)

///
/// The function returns an `Error` if any issues arise while reading from the object store or processing the data stream.
async fn find_first_newline(
    object_store: &Arc<dyn ObjectStore>,
    location: &Path,
    start: u64,
    end: u64,
    newline: u8,
) -> Result<u64> {
    let options = GetOptions {
        range: Some(GetRange::Bounded(start..end)),
        ..Default::default()
    };

    let result = object_store.get_opts(location, options).await?;
    let mut result_stream = result.into_stream();

    let mut index = 0;

    while let Some(chunk) = result_stream.next().await.transpose()? {
        if let Some(position) = chunk.iter().position(|&byte| byte == newline) {
            let position = position as u64;
            return Ok(index + position);
        }

        index += chunk.len() as u64;
    }

    Ok(index)
}

/// Generates test files with min-max statistics in different overlap patterns.
///