Backup the source directory to the target directory using rsync for incremental sync. Falls back to shutil.copytree if rsync is not available. Args: source_dir: The directory to backup (e.g., HOST_WORKSPACE). target_dir: The destination directory (e.g., inside outputs).
(
source_dir: Path,
target_dir: Path,
additional_ignores: Optional[List[str]] = None,
logger: Optional[Callable[[str], None]] = None,
)
| 24 | |
| 25 | |
| 26 | def backup_workspace( |
| 27 | source_dir: Path, |
| 28 | target_dir: Path, |
| 29 | additional_ignores: Optional[List[str]] = None, |
| 30 | logger: Optional[Callable[[str], None]] = None, |
| 31 | ) -> bool: |
| 32 | """ |
| 33 | Backup the source directory to the target directory using rsync for incremental sync. |
| 34 | Falls back to shutil.copytree if rsync is not available. |
| 35 | |
| 36 | Args: |
| 37 | source_dir: The directory to backup (e.g., HOST_WORKSPACE). |
| 38 | target_dir: The destination directory (e.g., inside outputs). |
| 39 | additional_ignores: List of patterns to ignore in addition to defaults. |
| 40 | logger: Optional logger callable for status messages. |
| 41 | |
| 42 | Returns: |
| 43 | True if backup succeeded, False otherwise. |
| 44 | """ |
| 45 | |
| 46 | def _log(msg: str) -> None: |
| 47 | try: |
| 48 | if logger: |
| 49 | logger(msg) |
| 50 | else: |
| 51 | print(msg) |
| 52 | except Exception: |
| 53 | print(msg) # Fallback if logger fails (e.g. log file path missing) |
| 54 | |
| 55 | source_dir = Path(source_dir) |
| 56 | target_dir = Path(target_dir) |
| 57 | |
| 58 | if not source_dir.exists(): |
| 59 | _log(f"[Checkpoint] Backup skipped: source {source_dir} does not exist") |
| 60 | return False |
| 61 | |
| 62 | # Ensure target parent exists |
| 63 | target_dir.parent.mkdir(parents=True, exist_ok=True) |
| 64 | |
| 65 | # Build ignore patterns |
| 66 | ignores = list(DEFAULT_IGNORES) |
| 67 | if additional_ignores: |
| 68 | ignores.extend(additional_ignores) |
| 69 | |
| 70 | # Try rsync first (incremental, faster for subsequent backups) |
| 71 | try: |
| 72 | # Build rsync command |
| 73 | # Use --no-owner --no-group to avoid permission errors when syncing |
| 74 | # files created by different users (e.g., docker container vs host). |
| 75 | # Use --ignore-errors so unreadable files (e.g. container-created) don't fail the whole backup. |
| 76 | cmd = [ |
| 77 | "rsync", |
| 78 | "-a", |
| 79 | "--delete", |
| 80 | "--no-owner", |
| 81 | "--no-group", |
| 82 | "--ignore-errors", |
| 83 | ] |