Given a file reader and a list of row groups, this is a generator of record batch generators (where each sub-generator is the contents of a single row group).
| 1204 | /// Given a file reader and a list of row groups, this is a generator of record |
| 1205 | /// batch generators (where each sub-generator is the contents of a single row group). |
| 1206 | class RowGroupGenerator { |
| 1207 | public: |
| 1208 | using RecordBatchGenerator = |
| 1209 | ::arrow::AsyncGenerator<std::shared_ptr<::arrow::RecordBatch>>; |
| 1210 | |
| 1211 | struct ReadRequest { |
| 1212 | ::arrow::Future<RecordBatchGenerator> read; |
| 1213 | int64_t num_rows; |
| 1214 | }; |
| 1215 | |
| 1216 | explicit RowGroupGenerator(std::shared_ptr<FileReaderImpl> arrow_reader, |
| 1217 | ::arrow::internal::Executor* cpu_executor, |
| 1218 | std::vector<int> row_groups, std::vector<int> column_indices, |
| 1219 | int64_t min_rows_in_flight) |
| 1220 | : arrow_reader_(std::move(arrow_reader)), |
| 1221 | cpu_executor_(cpu_executor), |
| 1222 | row_groups_(std::move(row_groups)), |
| 1223 | column_indices_(std::move(column_indices)), |
| 1224 | min_rows_in_flight_(min_rows_in_flight), |
| 1225 | rows_in_flight_(0), |
| 1226 | index_(0), |
| 1227 | readahead_index_(0) {} |
| 1228 | |
| 1229 | ::arrow::Future<RecordBatchGenerator> operator()() { |
| 1230 | if (index_ >= row_groups_.size()) { |
| 1231 | return ::arrow::AsyncGeneratorEnd<RecordBatchGenerator>(); |
| 1232 | } |
| 1233 | index_++; |
| 1234 | FillReadahead(); |
| 1235 | ReadRequest next = std::move(in_flight_reads_.front()); |
| 1236 | DCHECK(!in_flight_reads_.empty()); |
| 1237 | in_flight_reads_.pop(); |
| 1238 | rows_in_flight_ -= next.num_rows; |
| 1239 | return next.read; |
| 1240 | } |
| 1241 | |
| 1242 | private: |
| 1243 | void FillReadahead() { |
| 1244 | if (min_rows_in_flight_ == 0) { |
| 1245 | // No readahead, fetch the batch when it is asked for |
| 1246 | FetchNext(); |
| 1247 | } else { |
| 1248 | while (readahead_index_ < row_groups_.size() && |
| 1249 | rows_in_flight_ < min_rows_in_flight_) { |
| 1250 | FetchNext(); |
| 1251 | } |
| 1252 | } |
| 1253 | } |
| 1254 | |
| 1255 | void FetchNext() { |
| 1256 | size_t row_group_index = readahead_index_++; |
| 1257 | int row_group = row_groups_[row_group_index]; |
| 1258 | std::vector<int> column_indices = column_indices_; |
| 1259 | auto reader = arrow_reader_; |
| 1260 | int64_t num_rows = |
| 1261 | reader->parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); |
| 1262 | rows_in_flight_ += num_rows; |
| 1263 | ::arrow::Future<RecordBatchGenerator> row_group_read; |
no outgoing calls
no test coverage detected