MCPcopy Create free account
hub / github.com/apache/arrow / ReadRowGroups

Method ReadRowGroups

cpp/src/parquet/arrow/reader.cc:1362–1526  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1360}
1361
1362Result<std::shared_ptr<Table>> FileReaderImpl::ReadRowGroups(
1363 const std::vector<int>& row_groups, const std::vector<int>& column_indices) {
1364 RETURN_NOT_OK(BoundsCheck(row_groups, column_indices));
1365
1366 // PARQUET-1698/PARQUET-1820: pre-buffer row groups/column chunks if enabled
1367 if (reader_properties_.pre_buffer()) {
1368 BEGIN_PARQUET_CATCH_EXCEPTIONS
1369 parquet_reader()->PreBuffer(row_groups, column_indices,
1370 reader_properties_.io_context(),
1371 reader_properties_.cache_options());
1372 END_PARQUET_CATCH_EXCEPTIONS
1373 }
1374
1375 auto fut = DecodeRowGroups(/*self=*/nullptr, row_groups, column_indices,
1376 /*cpu_executor=*/nullptr);
1377 return fut.MoveResult();
1378}
1379
1380Future<std::shared_ptr<Table>> FileReaderImpl::DecodeRowGroups(
1381 std::shared_ptr<FileReaderImpl> self, const std::vector<int>& row_groups,
1382 const std::vector<int>& column_indices, ::arrow::internal::Executor* cpu_executor) {
1383 // `self` is used solely to keep `this` alive in an async context - but we use this
1384 // in a sync context too so use `this` over `self`
1385 std::vector<std::shared_ptr<ColumnReaderImpl>> readers;
1386 std::shared_ptr<::arrow::Schema> result_schema;
1387 std::vector<int> field_indices;
1388 RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &result_schema,
1389 &field_indices));
1390 // OptionalParallelForAsync requires an executor
1391 if (!cpu_executor) cpu_executor = ::arrow::internal::GetCpuThreadPool();
1392
1393 auto read_column = [field_indices, row_groups, self, this](
1394 size_t reader_index, std::shared_ptr<ColumnReaderImpl> reader)
1395 -> ::arrow::Result<std::shared_ptr<::arrow::ChunkedArray>> {
1396 std::shared_ptr<::arrow::ChunkedArray> column;
1397 RETURN_NOT_OK(
1398 ReadColumn(field_indices[reader_index], row_groups, reader.get(), &column));
1399 return column;
1400 };
1401 auto make_table = [result_schema, row_groups, self,
1402 this](const ::arrow::ChunkedArrayVector& columns)
1403 -> ::arrow::Result<std::shared_ptr<Table>> {
1404 int64_t num_rows = 0;
1405 if (!columns.empty()) {
1406 num_rows = columns[0]->length();
1407 } else {
1408 for (int i : row_groups) {
1409 num_rows += parquet_reader()->metadata()->RowGroup(i)->num_rows();
1410 }
1411 }
1412 auto table = Table::Make(std::move(result_schema), columns, num_rows);
1413 RETURN_NOT_OK(table->Validate());
1414 return table;
1415 };
1416 return ::arrow::internal::OptionalParallelForAsync(reader_properties_.use_threads(),
1417 std::move(readers), read_column,
1418 cpu_executor)
1419 .Then(std::move(make_table));

Calls 3

BoundsCheckFunction · 0.85
ARROW_ASSIGN_OR_RAISEFunction · 0.70
OKFunction · 0.50

Tested by

no test coverage detected