Function get_file_hash

modelscope/utils/file_utils.py:209–299

Compute SHA256 hash for a file path, bytes, or file-like object. Args: file_path_or_obj: File path, bytes, or file-like object. buffer_size_mb: Read buffer size in MB. Default 16MB. tqdm_desc: Progress bar description. disable_tqdm: Whether to disable progress ba

(
    file_path_or_obj: Union[str, Path, bytes, BinaryIO],
    buffer_size_mb: Optional[int] = 16,
    tqdm_desc: Optional[str] = '[Calculating]',
    disable_tqdm: Optional[bool] = True,
)

def get_file_hash(
    file_path_or_obj: Union[str, Path, bytes, BinaryIO],
    buffer_size_mb: Optional[int] = 16,
    tqdm_desc: Optional[str] = '[Calculating]',
    disable_tqdm: Optional[bool] = True,
) -> dict:
    """Compute SHA256 hash for a file path, bytes, or file-like object.

    Args:
        file_path_or_obj: File path, bytes, or file-like object.
        buffer_size_mb: Read buffer size in MB. Default 16MB.
        tqdm_desc: Progress bar description.
        disable_tqdm: Whether to disable progress bar.

    Returns:
        dict with keys: file_path_or_obj, file_hash, file_size.
    """
    from tqdm.auto import tqdm

    declared_size = get_file_size(file_path_or_obj)
    if declared_size > 1024 * 1024 * 1024:  # 1GB
        disable_tqdm = False
        name = 'Large File'
        if isinstance(file_path_or_obj, (str, Path)):
            path = file_path_or_obj if isinstance(
                file_path_or_obj, Path) else Path(file_path_or_obj)
            name = path.name
        tqdm_desc = f'[Validating Hash for {name}]'

    buffer_size = buffer_size_mb * 1024 * 1024
    file_hash = hashlib.sha256()

    progress = tqdm(
        total=declared_size,
        initial=0,
        unit_scale=True,
        dynamic_ncols=True,
        unit='B',
        desc=tqdm_desc,
        disable=disable_tqdm,
    )

    if isinstance(file_path_or_obj, (str, Path)):
        bytes_hashed = 0
        with open(file_path_or_obj, 'rb') as f:
            while byte_chunk := f.read(buffer_size):
                file_hash.update(byte_chunk)
                bytes_hashed += len(byte_chunk)
                progress.update(len(byte_chunk))
        file_hash = file_hash.hexdigest()
        if bytes_hashed != declared_size:
            logger.warning(
                f'File size changed during hash computation: '
                f'declared {declared_size} bytes, actually hashed {bytes_hashed} bytes. '
                f'File may have been modified: {file_path_or_obj}')
        file_size = bytes_hashed

    elif isinstance(file_path_or_obj, bytes):