MCPcopy Create free account
hub / github.com/apache/arrow / RowGroupGenerator

Class RowGroupGenerator

cpp/src/parquet/arrow/reader.cc:1206–1319  ·  view source on GitHub ↗

Given a file reader and a list of row groups, this is a generator of record batch generators (where each sub-generator is the contents of a single row group).

Source from the content-addressed store, hash-verified

1204/// Given a file reader and a list of row groups, this is a generator of record
1205/// batch generators (where each sub-generator is the contents of a single row group).
1206class RowGroupGenerator {
1207 public:
1208 using RecordBatchGenerator =
1209 ::arrow::AsyncGenerator<std::shared_ptr<::arrow::RecordBatch>>;
1210
1211 struct ReadRequest {
1212 ::arrow::Future<RecordBatchGenerator> read;
1213 int64_t num_rows;
1214 };
1215
1216 explicit RowGroupGenerator(std::shared_ptr<FileReaderImpl> arrow_reader,
1217 ::arrow::internal::Executor* cpu_executor,
1218 std::vector<int> row_groups, std::vector<int> column_indices,
1219 int64_t min_rows_in_flight)
1220 : arrow_reader_(std::move(arrow_reader)),
1221 cpu_executor_(cpu_executor),
1222 row_groups_(std::move(row_groups)),
1223 column_indices_(std::move(column_indices)),
1224 min_rows_in_flight_(min_rows_in_flight),
1225 rows_in_flight_(0),
1226 index_(0),
1227 readahead_index_(0) {}
1228
1229 ::arrow::Future<RecordBatchGenerator> operator()() {
1230 if (index_ >= row_groups_.size()) {
1231 return ::arrow::AsyncGeneratorEnd<RecordBatchGenerator>();
1232 }
1233 index_++;
1234 FillReadahead();
1235 ReadRequest next = std::move(in_flight_reads_.front());
1236 DCHECK(!in_flight_reads_.empty());
1237 in_flight_reads_.pop();
1238 rows_in_flight_ -= next.num_rows;
1239 return next.read;
1240 }
1241
1242 private:
1243 void FillReadahead() {
1244 if (min_rows_in_flight_ == 0) {
1245 // No readahead, fetch the batch when it is asked for
1246 FetchNext();
1247 } else {
1248 while (readahead_index_ < row_groups_.size() &&
1249 rows_in_flight_ < min_rows_in_flight_) {
1250 FetchNext();
1251 }
1252 }
1253 }
1254
1255 void FetchNext() {
1256 size_t row_group_index = readahead_index_++;
1257 int row_group = row_groups_[row_group_index];
1258 std::vector<int> column_indices = column_indices_;
1259 auto reader = arrow_reader_;
1260 int64_t num_rows =
1261 reader->parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
1262 rows_in_flight_ += num_rows;
1263 ::arrow::Future<RecordBatchGenerator> row_group_read;

Callers 1

Calls

no outgoing calls

Tested by

no test coverage detected