MCPcopy Create free account
hub / github.com/apache/arrow / LoadBatch

Method LoadBatch

cpp/src/parquet/arrow/reader.cc:498–553  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

496 bool IsOrHasRepeatedChild() const final { return false; }
497
498 Status LoadBatch(int64_t records_to_read) final {
499 BEGIN_PARQUET_CATCH_EXCEPTIONS
500 out_ = nullptr;
501 record_reader_->Reset();
502 // Pre-allocation gives much better performance for flat columns
503 record_reader_->Reserve(records_to_read);
504 const bool should_load_statistics = ctx_->reader_properties->should_load_statistics();
505 int64_t num_target_row_groups = 0;
506 while (records_to_read > 0) {
507 if (!record_reader_->HasMoreData()) {
508 break;
509 }
510 int64_t records_read = record_reader_->ReadRecords(records_to_read);
511 records_to_read -= records_read;
512 if (records_read == 0) {
513 NextRowGroup();
514 } else {
515 num_target_row_groups++;
516 // We can't mix multiple row groups when we load statistics
517 // because statistics are associated with a row group. If we
518 // want to mix multiple row groups and keep valid statistics,
519 // we need to implement a statistics merge logic.
520 if (should_load_statistics) {
521 break;
522 }
523 }
524 }
525 RETURN_NOT_OK(TransferColumnData(
526 record_reader_.get(),
527 num_target_row_groups == 1 ? input_->column_chunk_metadata() : nullptr, field_,
528 descr_, ctx_.get(), &out_));
529 return Status::OK();
530 END_PARQUET_CATCH_EXCEPTIONS
531 }
532
533 ::arrow::Status BuildArray(int64_t length_upper_bound,
534 std::shared_ptr<::arrow::ChunkedArray>* out) final {
535 *out = out_;
536 return Status::OK();
537 }
538
539 const std::shared_ptr<Field> field() override { return field_; }
540
541 private:
542 std::shared_ptr<ChunkedArray> out_;
543 void NextRowGroup() {
544 std::unique_ptr<PageReader> page_reader = input_->NextChunk();
545 record_reader_->SetPageReader(std::move(page_reader));
546 }
547
548 std::shared_ptr<ReaderContext> ctx_;
549 std::shared_ptr<Field> field_;
550 std::unique_ptr<FileColumnIterator> input_;
551 const ColumnDescriptor* descr_;
552 std::shared_ptr<RecordReader> record_reader_;
553};
554
555// Column reader for extension arrays

Callers 3

LoadBatchMethod · 0.45
LoadBatchMethod · 0.45
LoadBatchFunction · 0.45

Calls 8

TransferColumnDataFunction · 0.85
HasMoreDataMethod · 0.80
ReadRecordsMethod · 0.80
column_chunk_metadataMethod · 0.80
OKFunction · 0.50
ResetMethod · 0.45
ReserveMethod · 0.45
getMethod · 0.45

Tested by

no test coverage detected