Stage batches, popping and delivering batches if enough data has arrived
| 211 | |
| 212 | // Stage batches, popping and delivering batches if enough data has arrived |
| 213 | Status Push(std::shared_ptr<RecordBatch> batch) { |
| 214 | uint64_t delta_staged = batch->num_rows(); |
| 215 | rows_currently_staged_ += delta_staged; |
| 216 | staged_batches_.push_back(std::move(batch)); |
| 217 | while (!staged_batches_.empty() && |
| 218 | (writer_state_->StagingFull() || |
| 219 | rows_currently_staged_ >= options_.min_rows_per_group)) { |
| 220 | ARROW_ASSIGN_OR_RAISE(int64_t rows_popped, PopAndDeliverStagedBatch()); |
| 221 | delta_staged -= rows_popped; |
| 222 | } |
| 223 | // Note, delta_staged may be negative if we were able to deliver some data |
| 224 | writer_state_->staged_rows_count += delta_staged; |
| 225 | return Status::OK(); |
| 226 | } |
| 227 | |
| 228 | Status Finish() { |
| 229 | writer_state_->staged_rows_count -= rows_currently_staged_; |