MCPcopy Create free account
hub / github.com/apache/arrow / FullListAsync

Method FullListAsync

cpp/src/arrow/filesystem/s3fs.cc:2770–2796  ·  view source on GitHub ↗

Fully list all files from all buckets

Source from the content-addressed store, hash-verified

2768
2769 // Fully list all files from all buckets
2770 void FullListAsync(bool include_implicit_dirs, util::AsyncTaskScheduler* scheduler,
2771 FileInfoSink sink, bool recursive) {
2772 scheduler->AddSimpleTask(
2773 [this, scheduler, sink, include_implicit_dirs, recursive]() mutable {
2774 return ListBucketsAsync().Then(
2775 [this, scheduler, sink, include_implicit_dirs,
2776 recursive](const std::vector<std::string>& buckets) mutable {
2777 // Return the buckets themselves as directories
2778 std::vector<FileInfo> buckets_as_directories =
2779 MakeDirectoryInfos(buckets);
2780 sink.Push(std::move(buckets_as_directories));
2781
2782 if (recursive) {
2783 // Recursively list each bucket (these will run in parallel but sink
2784 // should be thread safe and so this is ok)
2785 for (const auto& bucket : buckets) {
2786 FileSelector select;
2787 select.allow_not_found = true;
2788 select.recursive = true;
2789 select.base_dir = bucket;
2790 ListAsync(select, bucket, "", include_implicit_dirs, scheduler, sink);
2791 }
2792 }
2793 });
2794 },
2795 std::string_view("FullListBucketScan"));
2796 }
2797
2798 // Delete multiple objects at once
2799 Future<> DeleteObjectsAsync(const std::string& bucket,

Callers 1

GetFileInfoGeneratorMethod · 0.95

Calls 2

ThenMethod · 0.80
PushMethod · 0.45

Tested by

no test coverage detected