Function download_gcs_folder

tensorflow_datasets/core/utils/gcs_utils.py:112–138

Downloads prepared GCS folder to local folder.

(
    gcs_folder: epath.Path,
    local_folder: epath.PathLike,
    max_simultaneous_downloads: int = 25,
)

def download_gcs_folder(
    gcs_folder: epath.Path,
    local_folder: epath.PathLike,
    max_simultaneous_downloads: int = 25,
) -> None:
  """Downloads prepared GCS folder to local folder."""
  if _is_gcs_disabled:
    raise AssertionError('Cannot download from GCS when _is_gcs_disabled')

  # Filter out the diffs folder if present
  paths_to_dl = [p for p in gcs_folder.iterdir() if p.name != 'diffs']

  with tqdm_utils.async_tqdm(
      total=len(paths_to_dl), desc='Dl Completed...', unit=' file'
  ) as pbar:

    def _copy(gcs_path_: epath.Path):
      # Copy 'gs://tfds-data/datasets/ds/1.0.0/file' -> `local_dir/file`
      gcs_path_.copy(dst=os.path.join(local_folder, gcs_path_.name))
      pbar.update(1)

    with concurrent.futures.ThreadPoolExecutor(
        max_workers=max_simultaneous_downloads
    ) as executor:
      futures = [executor.submit(_copy, path) for path in paths_to_dl]
      for future in concurrent.futures.as_completed(futures):
        future.result()


def download_gcs_dataset(