MCPcopy Create free account
hub / github.com/apache/paimon-rust / open_source_stream

Function open_source_stream

crates/paimon/src/table/data_evolution_reader.rs:480–533  ·  view source on GitHub ↗
(
    split: &DataSplit,
    source: &FieldSource,
    row_ranges: Option<Vec<RowRange>>,
    file_io: FileIO,
    schema_manager: SchemaManager,
    table_schema_id: i64,
    table_fields: Vec<DataFi

Source from the content-addressed store, hash-verified

478
479#[allow(clippy::too_many_arguments)]
480fn open_source_stream(
481 split: &DataSplit,
482 source: &FieldSource,
483 row_ranges: Option<Vec<RowRange>>,
484 file_io: FileIO,
485 schema_manager: SchemaManager,
486 table_schema_id: i64,
487 table_fields: Vec<DataField>,
488 blob_as_descriptor: bool,
489) -> crate::Result<ArrowRecordBatchStream> {
490 let file_reader = DataFileReader::new(
491 file_io,
492 schema_manager,
493 table_schema_id,
494 table_fields,
495 source.read_fields().to_vec(),
496 Vec::new(),
497 )
498 .with_blob_as_descriptor(blob_as_descriptor);
499
500 match source {
501 FieldSource::DataFile {
502 file, data_fields, ..
503 } => file_reader.read_single_file_stream(
504 split,
505 file.as_ref().clone(),
506 data_fields.clone(),
507 None,
508 row_ranges,
509 ),
510 FieldSource::BlobBunch {
511 bunch, data_fields, ..
512 } => {
513 let split = split.clone();
514 let files = bunch.files.clone();
515 let data_fields = data_fields.clone();
516 Ok(try_stream! {
517 for file in files {
518 let mut stream = file_reader.read_single_file_stream(
519 &split,
520 file,
521 data_fields.clone(),
522 None,
523 row_ranges.clone(),
524 )?;
525 while let Some(batch) = stream.next().await {
526 yield batch?;
527 }
528 }
529 }
530 .boxed())
531 }
532 }
533}
534
535#[derive(Debug, Clone)]
536struct PreparedMergeGroup {

Callers

nothing calls this directly

Calls 4

to_vecMethod · 0.80
read_fieldsMethod · 0.80

Tested by

no test coverage detected