MCPcopy Create free account
hub / github.com/apache/arrow / DecodeRowGroups

Method DecodeRowGroups

cpp/src/parquet/arrow/reader.cc:1380–1420  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1378}
1379
1380Future<std::shared_ptr<Table>> FileReaderImpl::DecodeRowGroups(
1381 std::shared_ptr<FileReaderImpl> self, const std::vector<int>& row_groups,
1382 const std::vector<int>& column_indices, ::arrow::internal::Executor* cpu_executor) {
1383 // `self` is used solely to keep `this` alive in an async context - but we use this
1384 // in a sync context too so use `this` over `self`
1385 std::vector<std::shared_ptr<ColumnReaderImpl>> readers;
1386 std::shared_ptr<::arrow::Schema> result_schema;
1387 std::vector<int> field_indices;
1388 RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &result_schema,
1389 &field_indices));
1390 // OptionalParallelForAsync requires an executor
1391 if (!cpu_executor) cpu_executor = ::arrow::internal::GetCpuThreadPool();
1392
1393 auto read_column = [field_indices, row_groups, self, this](
1394 size_t reader_index, std::shared_ptr<ColumnReaderImpl> reader)
1395 -> ::arrow::Result<std::shared_ptr<::arrow::ChunkedArray>> {
1396 std::shared_ptr<::arrow::ChunkedArray> column;
1397 RETURN_NOT_OK(
1398 ReadColumn(field_indices[reader_index], row_groups, reader.get(), &column));
1399 return column;
1400 };
1401 auto make_table = [result_schema, row_groups, self,
1402 this](const ::arrow::ChunkedArrayVector& columns)
1403 -> ::arrow::Result<std::shared_ptr<Table>> {
1404 int64_t num_rows = 0;
1405 if (!columns.empty()) {
1406 num_rows = columns[0]->length();
1407 } else {
1408 for (int i : row_groups) {
1409 num_rows += parquet_reader()->metadata()->RowGroup(i)->num_rows();
1410 }
1411 }
1412 auto table = Table::Make(std::move(result_schema), columns, num_rows);
1413 RETURN_NOT_OK(table->Validate());
1414 return table;
1415 };
1416 return ::arrow::internal::OptionalParallelForAsync(reader_properties_.use_threads(),
1417 std::move(readers), read_column,
1418 cpu_executor)
1419 .Then(std::move(make_table));
1420}
1421
1422std::shared_ptr<RowGroupReader> FileReaderImpl::RowGroup(int row_group_index) {
1423 return std::make_shared<RowGroupReaderImpl>(this, row_group_index);

Callers 1

ReadOneRowGroupMethod · 0.80

Calls 12

GetCpuThreadPoolFunction · 0.85
OptionalParallelForAsyncFunction · 0.85
ThenMethod · 0.80
MakeFunction · 0.50
getMethod · 0.45
emptyMethod · 0.45
lengthMethod · 0.45
num_rowsMethod · 0.45
RowGroupMethod · 0.45
metadataMethod · 0.45
ValidateMethod · 0.45
use_threadsMethod · 0.45

Tested by

no test coverage detected