| 165 | } |
| 166 | |
| 167 | Result<std::shared_ptr<RecordBatch>> PopStagedBatch() { |
| 168 | std::vector<std::shared_ptr<RecordBatch>> batches_to_write; |
| 169 | uint64_t num_rows = 0; |
| 170 | while (!staged_batches_.empty()) { |
| 171 | std::shared_ptr<RecordBatch> next = std::move(staged_batches_.front()); |
| 172 | staged_batches_.pop_front(); |
| 173 | if (num_rows + next->num_rows() <= options_.max_rows_per_group) { |
| 174 | num_rows += next->num_rows(); |
| 175 | batches_to_write.push_back(std::move(next)); |
| 176 | if (num_rows == options_.max_rows_per_group) { |
| 177 | break; |
| 178 | } |
| 179 | } else { |
| 180 | uint64_t remaining = options_.max_rows_per_group - num_rows; |
| 181 | std::shared_ptr<RecordBatch> next_partial = |
| 182 | next->Slice(0, static_cast<int64_t>(remaining)); |
| 183 | batches_to_write.push_back(std::move(next_partial)); |
| 184 | std::shared_ptr<RecordBatch> next_remainder = |
| 185 | next->Slice(static_cast<int64_t>(remaining)); |
| 186 | staged_batches_.push_front(std::move(next_remainder)); |
| 187 | break; |
| 188 | } |
| 189 | } |
| 190 | DCHECK_GT(batches_to_write.size(), 0); |
| 191 | ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, |
| 192 | Table::FromRecordBatches(batches_to_write)); |
| 193 | return table->CombineChunksToBatch(); |
| 194 | } |
| 195 | |
| 196 | void ScheduleBatch(std::shared_ptr<RecordBatch> batch) { |
| 197 | file_tasks_->AddSimpleTask( |
nothing calls this directly
no test coverage detected