MCPcopy Create free account
hub / github.com/apache/arrow / GetRecordBatchReader

Method GetRecordBatchReader

cpp/src/parquet/arrow/reader.cc:1111–1202  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1109} // namespace
1110
1111Result<std::unique_ptr<RecordBatchReader>> FileReaderImpl::GetRecordBatchReader(
1112 const std::vector<int>& row_groups, const std::vector<int>& column_indices) {
1113 RETURN_NOT_OK(BoundsCheck(row_groups, column_indices));
1114
1115 if (reader_properties_.pre_buffer()) {
1116 // PARQUET-1698/PARQUET-1820: pre-buffer row groups/column chunks if enabled
1117 BEGIN_PARQUET_CATCH_EXCEPTIONS
1118 reader_->PreBuffer(row_groups, column_indices, reader_properties_.io_context(),
1119 reader_properties_.cache_options());
1120 END_PARQUET_CATCH_EXCEPTIONS
1121 }
1122
1123 std::vector<std::shared_ptr<ColumnReaderImpl>> readers;
1124 std::shared_ptr<::arrow::Schema> batch_schema;
1125 std::vector<int> field_indices;
1126 RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &batch_schema,
1127 &field_indices));
1128
1129 if (readers.empty()) {
1130 // Just generate all batches right now; they're cheap since they have no columns.
1131 int64_t batch_size = properties().batch_size();
1132 auto max_sized_batch =
1133 ::arrow::RecordBatch::Make(batch_schema, batch_size, ::arrow::ArrayVector{});
1134
1135 ::arrow::RecordBatchVector batches;
1136
1137 for (int row_group : row_groups) {
1138 int64_t num_rows = parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
1139
1140 batches.insert(batches.end(), static_cast<size_t>(num_rows / batch_size),
1141 max_sized_batch);
1142
1143 if (int64_t trailing_rows = num_rows % batch_size) {
1144 batches.push_back(max_sized_batch->Slice(0, trailing_rows));
1145 }
1146 }
1147
1148 return std::make_unique<RowGroupRecordBatchReader>(
1149 ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema));
1150 }
1151
1152 int64_t num_rows = 0;
1153 for (int row_group : row_groups) {
1154 num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
1155 }
1156
1157 using ::arrow::RecordBatchIterator;
1158
1159 // NB: This lambda will be invoked outside the scope of this call to
1160 // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value.
1161 // `this` is a non-owning pointer so we are relying on the parent FileReader outliving
1162 // this RecordBatchReader.
1163 ::arrow::Iterator<RecordBatchIterator> batches = ::arrow::MakeFunctionIterator(
1164 [readers, batch_schema, num_rows,
1165 this]() mutable -> ::arrow::Result<RecordBatchIterator> {
1166 ::arrow::ChunkedArrayVector columns(readers.size());
1167
1168 // don't reserve more rows than necessary

Callers 3

DoRoundTripWithBatchesFunction · 0.45
TestGetRecordBatchReaderFunction · 0.45
ReadInBatchesFunction · 0.45

Calls 15

BoundsCheckFunction · 0.85
MakeVectorIteratorFunction · 0.85
MakeFunctionIteratorFunction · 0.85
OptionalParallelForFunction · 0.85
MakeFlattenIteratorFunction · 0.85
push_backMethod · 0.80
NextBatchMethod · 0.80
MakeFunction · 0.50
EndFunction · 0.50
InvalidFunction · 0.50
emptyMethod · 0.45
num_rowsMethod · 0.45

Tested by 2

DoRoundTripWithBatchesFunction · 0.36
TestGetRecordBatchReaderFunction · 0.36