Function cached_path

src/datasets/utils/file_utils.py:134–254

Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Return: Local path (

(
    url_or_filename,
    download_config=None,
    **download_kwargs,
)

def cached_path(
    url_or_filename,
    download_config=None,
    **download_kwargs,
) -> str:
    """
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.

    Return:
        Local path (string)

    Raises:
        FileNotFoundError: in case of non-recoverable file
            (non-existent or no cache on disk)
        ConnectionError: in case of unreachable url
            and no cache on disk
        ValueError: if it couldn't parse the url or filename correctly
        httpx.NetworkError or requests.exceptions.ConnectionError: in case of internet connection issue
    """
    if download_config is None:
        download_config = DownloadConfig(**download_kwargs)

    cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH
    if isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)
    if isinstance(url_or_filename, Path):
        url_or_filename = str(url_or_filename)

    # Convert fsspec URL in the format "file://local/path" to "local/path"
    if can_be_local(url_or_filename):
        url_or_filename = strip_protocol(url_or_filename)

    if is_remote_url(url_or_filename):
        # URL, so get it from the cache (downloading if necessary)
        url_or_filename, storage_options = _prepare_path_and_storage_options(
            url_or_filename, download_config=download_config
        )
        # Download files from Hugging Face.
        # Note: no need to check for https://huggingface.co file URLs since _prepare_path_and_storage_options
        # prepares Hugging Face HTTP URLs as hf:// paths already
        if url_or_filename.startswith("hf://") and not url_or_filename.startswith("hf://buckets/"):
            resolved_path = huggingface_hub.HfFileSystem(
                endpoint=config.HF_ENDPOINT, token=download_config.token
            ).resolve_path(url_or_filename)
            try:
                output_path = huggingface_hub.HfApi(
                    endpoint=config.HF_ENDPOINT,
                    token=download_config.token,
                    library_name="datasets",
                    library_version=__version__,
                    user_agent=get_datasets_user_agent(download_config.user_agent),
                ).hf_hub_download(
                    repo_id=resolved_path.repo_id,
                    repo_type=resolved_path.repo_type,
                    revision=resolved_path.revision,