Method ListAsync

cpp/src/arrow/filesystem/s3fs.cc:2750–2765

Lists all file, potentially recursively, in a bucket include_implicit_dirs controls whether or not implicit directories should be included. These are directories that are not actually file objects but instead are inferred from other objects. For example, if a file exists with path A/B/C then implicit directories A/ and A/B/ will exist even if there are no file objects with these paths.

  // For example, if a file exists with path A/B/C then implicit directories A/ and A/B/
  // will exist even if there are no file objects with these paths.
  void ListAsync(const FileSelector& select, const std::string& bucket,
                 const std::string& key, bool include_implicit_dirs,
                 util::AsyncTaskScheduler* scheduler, FileInfoSink sink) {
    // We can only fetch kListObjectsMaxKeys files at a time and so we create a
    // scheduler and schedule a task to grab the first batch.  Once that's done we
    // schedule a new task for the next batch.  All of these tasks share the same
    // FileListerState object but none of these tasks run in parallel so there is
    // no need to worry about mutexes
    auto state = std::make_shared<FileListerState>(sink, select, bucket, key,
                                                   include_implicit_dirs, io_context_,
                                                   this->holder_.get());

    // Create the first file lister task (it may spawn more)
    auto file_lister_task = std::make_unique<FileListerTask>(state, scheduler);
    scheduler->AddTask(std::move(file_lister_task));
  }

  // Fully list all files from all buckets
  void FullListAsync(bool include_implicit_dirs, util::AsyncTaskScheduler* scheduler,