MCPcopy Create free account
hub / github.com/apache/arrow / GetRecordBatchGenerator

Method GetRecordBatchGenerator

cpp/src/arrow/ipc/reader.cc:1568–1597  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1566 }
1567
1568 Result<AsyncGenerator<std::shared_ptr<RecordBatch>>> GetRecordBatchGenerator(
1569 const bool coalesce, const io::IOContext& io_context,
1570 const io::CacheOptions cache_options,
1571 arrow::internal::Executor* executor) override {
1572 auto state = std::dynamic_pointer_cast<RecordBatchFileReaderImpl>(shared_from_this());
1573 // Prebuffering causes us to use a lot of futures which, at the moment,
1574 // can only slow things down when we are doing zero-copy in-memory reads.
1575 //
1576 // Prebuffering's read patterns are also slightly worse than the alternative
1577 // when doing whole-file reads because the logic is not in place to recognize
1578 // we can just read the entire file up-front
1579 if (!options_.included_fields.empty() &&
1580 options_.included_fields.size() != schema_->fields().size() &&
1581 !file_->supports_zero_copy()) {
1582 RETURN_NOT_OK(state->PreBufferMetadata({}));
1583 return SelectiveIpcFileRecordBatchGenerator(std::move(state));
1584 }
1585
1586 std::shared_ptr<io::internal::ReadRangeCache> cached_source;
1587 if (coalesce && !file_->supports_zero_copy()) {
1588 if (!owned_file_) return Status::Invalid("Cannot coalesce without an owned file");
1589 // Since the user is asking for all fields then we can cache the entire
1590 // file (up to the footer)
1591 cached_source = std::make_shared<io::internal::ReadRangeCache>(file_, io_context,
1592 cache_options);
1593 RETURN_NOT_OK(cached_source->Cache({{0, footer_offset_}}));
1594 }
1595 return WholeIpcFileRecordBatchGenerator(std::move(state), std::move(cached_source),
1596 io_context, executor);
1597 }
1598
1599 Status DoPreBufferMetadata(const std::vector<int>& indices) {
1600 RETURN_NOT_OK(CacheMetadata(indices));

Callers 3

ScanBatchesAsyncMethod · 0.45
ReadBatchesMethod · 0.45
TEST_FFunction · 0.45

Calls 9

PreBufferMetadataMethod · 0.80
InvalidFunction · 0.50
emptyMethod · 0.45
sizeMethod · 0.45
fieldsMethod · 0.45
supports_zero_copyMethod · 0.45
CacheMethod · 0.45

Tested by 2

ReadBatchesMethod · 0.36
TEST_FFunction · 0.36