Check if lock is stale and remove if so. A lock is treated as stale if either: 1. ALL holders have dead PIDs (process crashed), OR 2. ANY holder has held the lock past _max_hold_seconds() — even alive-but-wedged processes get stolen so subsequent runs aren't
(self)
| 145 | return _DEFAULT_MAX_HOLD_S |
| 146 | |
| 147 | def _check_stale_lock(self) -> bool: |
| 148 | """Check if lock is stale and remove if so. |
| 149 | |
| 150 | A lock is treated as stale if either: |
| 151 | 1. ALL holders have dead PIDs (process crashed), OR |
| 152 | 2. ANY holder has held the lock past _max_hold_seconds() — even |
| 153 | alive-but-wedged processes get stolen so subsequent runs aren't |
| 154 | blocked indefinitely. The wedged process gets a deadman force-exit |
| 155 | from test.py's watchdog separately. |
| 156 | |
| 157 | Uses psutil for process liveness checks (more reliable than kernel32 on Windows). |
| 158 | |
| 159 | Returns: |
| 160 | True if lock was stale and removed, False otherwise |
| 161 | """ |
| 162 | try: |
| 163 | holders = self._db.get_lock_info(self._lock_name) |
| 164 | if not holders: |
| 165 | return False |
| 166 | |
| 167 | # Check all holders with psutil (more reliable than kernel32) |
| 168 | all_dead = all( |
| 169 | not _is_process_alive_psutil(h["owner_pid"]) for h in holders |
| 170 | ) |
| 171 | |
| 172 | # Check for alive-but-wedged: any holder past max-hold threshold. |
| 173 | now = time.time() |
| 174 | max_hold = self._max_hold_seconds() |
| 175 | wedged_holders = [h for h in holders if (now - h["acquired_at"]) > max_hold] |
| 176 | |
| 177 | if not all_dead and not wedged_holders: |
| 178 | return False |
| 179 | |
| 180 | if not BuildLock._stale_lock_warned: |
| 181 | BuildLock._stale_lock_warned = True |
| 182 | if all_dead: |
| 183 | dead_pids = [h["owner_pid"] for h in holders] |
| 184 | print( |
| 185 | f"Detected stale lock '{self._lock_name}' (dead PIDs: {dead_pids}). Removing..." |
| 186 | ) |
| 187 | else: |
| 188 | parts = [ |
| 189 | f"PID {h['owner_pid']} ({(now - h['acquired_at']):.0f}s)" |
| 190 | for h in wedged_holders |
| 191 | ] |
| 192 | print( |
| 193 | f"Detected wedged lock '{self._lock_name}' " |
| 194 | f"(holder(s) past {max_hold:.0f}s max-hold: {', '.join(parts)}). Stealing..." |
| 195 | ) |
| 196 | |
| 197 | # Force-break: either all holders are dead, or one is wedged past max-hold. |
| 198 | if self._db.force_break(self._lock_name): |
| 199 | print(f"Removed stale lock: {self._lock_name}") |
| 200 | BuildLock._stale_lock_warned = False |
| 201 | return True |
| 202 | |
| 203 | return False |
| 204 | except KeyboardInterrupt as ki: |