| 1109 | } // namespace |
| 1110 | |
| 1111 | Result<std::unique_ptr<RecordBatchReader>> FileReaderImpl::GetRecordBatchReader( |
| 1112 | const std::vector<int>& row_groups, const std::vector<int>& column_indices) { |
| 1113 | RETURN_NOT_OK(BoundsCheck(row_groups, column_indices)); |
| 1114 | |
| 1115 | if (reader_properties_.pre_buffer()) { |
| 1116 | // PARQUET-1698/PARQUET-1820: pre-buffer row groups/column chunks if enabled |
| 1117 | BEGIN_PARQUET_CATCH_EXCEPTIONS |
| 1118 | reader_->PreBuffer(row_groups, column_indices, reader_properties_.io_context(), |
| 1119 | reader_properties_.cache_options()); |
| 1120 | END_PARQUET_CATCH_EXCEPTIONS |
| 1121 | } |
| 1122 | |
| 1123 | std::vector<std::shared_ptr<ColumnReaderImpl>> readers; |
| 1124 | std::shared_ptr<::arrow::Schema> batch_schema; |
| 1125 | std::vector<int> field_indices; |
| 1126 | RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &batch_schema, |
| 1127 | &field_indices)); |
| 1128 | |
| 1129 | if (readers.empty()) { |
| 1130 | // Just generate all batches right now; they're cheap since they have no columns. |
| 1131 | int64_t batch_size = properties().batch_size(); |
| 1132 | auto max_sized_batch = |
| 1133 | ::arrow::RecordBatch::Make(batch_schema, batch_size, ::arrow::ArrayVector{}); |
| 1134 | |
| 1135 | ::arrow::RecordBatchVector batches; |
| 1136 | |
| 1137 | for (int row_group : row_groups) { |
| 1138 | int64_t num_rows = parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); |
| 1139 | |
| 1140 | batches.insert(batches.end(), static_cast<size_t>(num_rows / batch_size), |
| 1141 | max_sized_batch); |
| 1142 | |
| 1143 | if (int64_t trailing_rows = num_rows % batch_size) { |
| 1144 | batches.push_back(max_sized_batch->Slice(0, trailing_rows)); |
| 1145 | } |
| 1146 | } |
| 1147 | |
| 1148 | return std::make_unique<RowGroupRecordBatchReader>( |
| 1149 | ::arrow::MakeVectorIterator(std::move(batches)), std::move(batch_schema)); |
| 1150 | } |
| 1151 | |
| 1152 | int64_t num_rows = 0; |
| 1153 | for (int row_group : row_groups) { |
| 1154 | num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); |
| 1155 | } |
| 1156 | |
| 1157 | using ::arrow::RecordBatchIterator; |
| 1158 | |
| 1159 | // NB: This lambda will be invoked outside the scope of this call to |
| 1160 | // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. |
| 1161 | // `this` is a non-owning pointer so we are relying on the parent FileReader outliving |
| 1162 | // this RecordBatchReader. |
| 1163 | ::arrow::Iterator<RecordBatchIterator> batches = ::arrow::MakeFunctionIterator( |
| 1164 | [readers, batch_schema, num_rows, |
| 1165 | this]() mutable -> ::arrow::Result<RecordBatchIterator> { |
| 1166 | ::arrow::ChunkedArrayVector columns(readers.size()); |
| 1167 | |
| 1168 | // don't reserve more rows than necessary |