MCPcopy Create free account
hub / github.com/apache/arrow / WriteBatch

Function WriteBatch

cpp/src/arrow/dataset/file_base.cc:375–403  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

373namespace {
374
375Status WriteBatch(
376 std::shared_ptr<RecordBatch> batch, compute::Expression guarantee,
377 FileSystemDatasetWriteOptions write_options,
378 std::function<Status(std::shared_ptr<RecordBatch>, const PartitionPathFormat&)>
379 write) {
380 ARROW_ASSIGN_OR_RAISE(auto groups, write_options.partitioning->Partition(batch));
381 batch.reset(); // drop to hopefully conserve memory
382
383 if (write_options.max_partitions <= 0) {
384 return Status::Invalid("max_partitions must be positive (was ",
385 write_options.max_partitions, ")");
386 }
387
388 if (groups.batches.size() > static_cast<size_t>(write_options.max_partitions)) {
389 return Status::Invalid("Fragment would be written into ", groups.batches.size(),
390 " partitions. This exceeds the maximum of ",
391 write_options.max_partitions);
392 }
393
394 for (std::size_t index = 0; index < groups.batches.size(); index++) {
395 auto partition_expression = and_(groups.expressions[index], guarantee);
396 auto next_batch = groups.batches[index];
397 PartitionPathFormat destination;
398 ARROW_ASSIGN_OR_RAISE(destination,
399 write_options.partitioning->Format(partition_expression));
400 RETURN_NOT_OK(write(next_batch, destination));
401 }
402 return Status::OK();
403}
404
405class DatasetWritingSinkNodeConsumer : public acero::SinkNodeConsumer {
406 public:

Callers 2

WriteNextBatchMethod · 0.70
WriteNextBatchMethod · 0.70

Calls 7

and_Function · 0.85
FormatMethod · 0.80
InvalidFunction · 0.50
ARROW_ASSIGN_OR_RAISEFunction · 0.50
OKFunction · 0.50
resetMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected