MCPcopy Create free account
hub / github.com/apache/arrow / DeleteObjectsAsync

Method DeleteObjectsAsync

cpp/src/arrow/filesystem/s3fs.cc:2799–2852  ·  view source on GitHub ↗

Delete multiple objects at once

Source from the content-addressed store, hash-verified

2797
2798 // Delete multiple objects at once
2799 Future<> DeleteObjectsAsync(const std::string& bucket,
2800 const std::vector<std::string>& keys) {
2801 struct DeleteCallback {
2802 std::string bucket;
2803
2804 Status operator()(const S3Model::DeleteObjectsOutcome& outcome) const {
2805 if (!outcome.IsSuccess()) {
2806 return ErrorToStatus("DeleteObjects", outcome.GetError());
2807 }
2808 // Also need to check per-key errors, even on successful outcome
2809 // See
2810 // https://docs.aws.amazon.com/fr_fr/AmazonS3/latest/API/multiobjectdeleteapi.html
2811 const auto& errors = outcome.GetResult().GetErrors();
2812 if (!errors.empty()) {
2813 std::stringstream ss;
2814 ss << "Got the following " << errors.size()
2815 << " errors when deleting objects in S3 bucket '" << bucket << "':\n";
2816 for (const auto& error : errors) {
2817 ss << "- key '" << error.GetKey() << "': " << error.GetMessage() << "\n";
2818 }
2819 return Status::IOError(ss.str());
2820 }
2821 return Status::OK();
2822 }
2823 };
2824
2825 const auto chunk_size = static_cast<size_t>(kMultipleDeleteMaxKeys);
2826 const DeleteCallback delete_cb{bucket};
2827
2828 std::vector<Future<>> futures;
2829 futures.reserve(bit_util::CeilDiv(keys.size(), chunk_size));
2830
2831 for (size_t start = 0; start < keys.size(); start += chunk_size) {
2832 S3Model::DeleteObjectsRequest req;
2833 S3Model::Delete del;
2834 size_t remaining = keys.size() - start;
2835 size_t next_chunk_size = std::min(remaining, chunk_size);
2836 for (size_t i = start; i < start + next_chunk_size; ++i) {
2837 del.AddObjects(S3Model::ObjectIdentifier().WithKey(ToAwsString(keys[i])));
2838 }
2839 req.SetBucket(ToAwsString(bucket));
2840 req.SetDelete(std::move(del));
2841 ARROW_ASSIGN_OR_RAISE(
2842 auto fut,
2843 SubmitIO(io_context_,
2844 [holder = holder_, req = std::move(req), delete_cb]() -> Status {
2845 ARROW_ASSIGN_OR_RAISE(auto client_lock, holder->Lock());
2846 return delete_cb(client_lock.Move()->DeleteObjects(req));
2847 }));
2848 futures.push_back(std::move(fut));
2849 }
2850
2851 return AllFinished(futures);
2852 }
2853
2854 Status DeleteObjects(const std::string& bucket, const std::vector<std::string>& keys) {
2855 return DeleteObjectsAsync(bucket, keys).status();

Callers 1

Calls 6

CeilDivFunction · 0.85
ToAwsStringFunction · 0.85
AllFinishedFunction · 0.85
push_backMethod · 0.80
reserveMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected