(
package: ModelPackage,
models_root: Path,
overwrite: bool,
source_dir_arg: str | None,
variant_arg: str | None,
)
| 1566 | chunk = response.read(1 << 20) |
| 1567 | if not chunk: |
| 1568 | break |
| 1569 | handle.write(chunk) |
| 1570 | written += len(chunk) |
| 1571 | if expected_size is not None and written != expected_size: |
| 1572 | raise RuntimeError(f"downloaded size mismatch for {target}: {written} != {expected_size}") |
| 1573 | return written |
| 1574 | |
| 1575 | |
| 1576 | def validate_required_files(package: ModelPackage, root: Path) -> None: |
| 1577 | missing = [relative for relative in package.required_files if not (root / relative).exists()] |
| 1578 | if missing: |
| 1579 | raise RuntimeError(f"installed package is missing required files: {missing}") |
| 1580 | |
| 1581 | |
| 1582 | def validate_composite_required_files(package: ModelPackage, staged_root: Path, final_root: Path) -> None: |
| 1583 | missing = [ |
| 1584 | relative |
| 1585 | for relative in package.required_files |
| 1586 | if not normalized_join(staged_root, relative).exists() and not normalized_join(final_root, relative).exists() |
| 1587 | ] |
| 1588 | if missing: |
| 1589 | raise RuntimeError(f"installed package is missing required files: {missing}") |
| 1590 | |
| 1591 | |
| 1592 | def validate_required_files_list(required_files: Iterable[str], root: Path, label: str) -> None: |
| 1593 | missing = [relative for relative in required_files if not (root / relative).exists()] |
| 1594 | if missing: |
| 1595 | raise RuntimeError(f"{label} is missing required files: {missing}") |
| 1596 | |
| 1597 | |
| 1598 | def install_snapshot_into_dir( |
| 1599 | source: SnapshotSource, |
| 1600 | destination_root: Path, |
| 1601 | required_files: Iterable[str], |
| 1602 | *, |
| 1603 | validate: bool = True, |
| 1604 | ) -> None: |
| 1605 | files = list_hf_files(source) |
| 1606 | for remote, relative, expected_size in files: |
| 1607 | destination = destination_root / relative |
| 1608 | destination.parent.mkdir(parents=True, exist_ok=True) |
| 1609 | print(f"download {remote}") |
| 1610 | download_file(hf_resolve_url(source, remote), destination, expected_size) |
| 1611 | if validate: |
| 1612 | validate_required_files_list(required_files, destination_root, source.repo_id) |
| 1613 | |
| 1614 | |
| 1615 | def write_vevo2_whisper_stats(target_path: Path, source_file: Path) -> None: |
| 1616 | payload = torch.load(source_file, map_location="cpu", weights_only=False) |
| 1617 | state = checkpoint_state_dict(payload) |
| 1618 | tensors = tensor_state_dict(state) |
| 1619 | if set(tensors.keys()) != {"mean", "std"}: |
| 1620 | raise RuntimeError(f"Vevo2 whisper stats must contain mean/std only: {sorted(tensors.keys())}") |
| 1621 | save_file(tensors, str(target_path)) |
| 1622 | |
| 1623 | |
| 1624 | def whisper_dims(payload: dict[str, Any]) -> dict[str, int]: |
| 1625 | raw_dims = payload.get("dims") |
no test coverage detected