MCPcopy Create free account
hub / github.com/apache/arrow / ReadBatches

Method ReadBatches

cpp/src/arrow/ipc/read_write_test.cc:1275–1328  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1273template <bool kCoalesce>
1274struct FileGeneratorWriterHelper : public FileWriterHelper {
1275 Status ReadBatches(const IpcReadOptions& options, RecordBatchVector* out_batches,
1276 ReadStats* out_stats = nullptr,
1277 MetadataVector* out_metadata_list = nullptr) override {
1278 // The generator doesn't track stats.
1279 EXPECT_EQ(nullptr, out_stats);
1280
1281 auto read_batches = [&](bool pre_buffer) -> Result<RecordBatchVector> {
1282 std::shared_ptr<io::RandomAccessFile> buf_reader;
1283 if (kCoalesce) {
1284 // Use a non-zero-copy enabled BufferReader so we can test paths properly
1285 buf_reader = std::make_shared<NoZeroCopyBufferReader>(buffer_);
1286 } else {
1287 buf_reader = std::make_shared<io::BufferReader>(buffer_);
1288 }
1289 AsyncGenerator<std::shared_ptr<RecordBatch>> generator;
1290
1291 {
1292 auto fut = RecordBatchFileReader::OpenAsync(buf_reader, footer_offset_, options);
1293 ARROW_ASSIGN_OR_RAISE(auto reader, fut.result());
1294 EXPECT_EQ(num_batches_written_, reader->num_record_batches());
1295 if (pre_buffer) {
1296 RETURN_NOT_OK(reader->PreBufferMetadata(/*indices=*/{}));
1297 }
1298 // Generator will keep reader alive internally
1299 ARROW_ASSIGN_OR_RAISE(generator, reader->GetRecordBatchGenerator(kCoalesce));
1300 }
1301
1302 // Generator is async-reentrant
1303 std::vector<Future<std::shared_ptr<RecordBatch>>> futures;
1304 for (int i = 0; i < num_batches_written_; ++i) {
1305 futures.push_back(generator());
1306 }
1307 auto fut = generator();
1308 ARROW_ASSIGN_OR_RAISE(auto final_batch, fut.result());
1309 EXPECT_EQ(nullptr, final_batch);
1310
1311 RecordBatchVector batches;
1312 for (auto& future : futures) {
1313 ARROW_ASSIGN_OR_RAISE(auto batch, future.result());
1314 EXPECT_NE(nullptr, batch);
1315 batches.push_back(batch);
1316 }
1317 return batches;
1318 };
1319
1320 ARROW_ASSIGN_OR_RAISE(*out_batches, read_batches(/*pre_buffer=*/false));
1321 // Also read with pre-buffered metadata, and check the results are equal
1322 ARROW_ASSIGN_OR_RAISE(auto batches_pre_buffered, read_batches(/*pre_buffer=*/true));
1323 for (int i = 0; i < num_batches_written_; ++i) {
1324 AssertBatchesEqual(*batches_pre_buffered[i], *(*out_batches)[i],
1325 /*check_metadata=*/true);
1326 }
1327 return Status::OK();
1328 }
1329};
1330
1331struct StreamWriterHelper {

Callers

nothing calls this directly

Calls 7

AssertBatchesEqualFunction · 0.85
PreBufferMetadataMethod · 0.80
push_backMethod · 0.80
ARROW_ASSIGN_OR_RAISEFunction · 0.70
OKFunction · 0.50
num_record_batchesMethod · 0.45

Tested by

no test coverage detected