| 224 | |
| 225 | |
| 226 | def make_watch_dog_thread( |
| 227 | seconds: int, |
| 228 | ) -> threading.Thread: # 60 seconds default timeout |
| 229 | def deadman_timer() -> None: |
| 230 | # Unconditional second-tier kill: if the primary watchdog gets stuck |
| 231 | # in dump_thread_stacks() / psutil queries / blocked I/O, this fires |
| 232 | # _DEADMAN_GRACE_S later and exits without running any diagnostics. |
| 233 | time.sleep(seconds + _DEADMAN_GRACE_S) |
| 234 | if _CANCEL_WATCHDOG.is_set(): |
| 235 | return |
| 236 | try: |
| 237 | _release_held_build_locks() |
| 238 | except KeyboardInterrupt as ki: |
| 239 | # Suppress: the unconditional os._exit() below is the entire point |
| 240 | # of the deadman timer. Re-raising would defeat it. |
| 241 | handle_keyboard_interrupt(ki) |
| 242 | except Exception: |
| 243 | pass |
| 244 | os._exit(3) # Exit code 3 = deadman fired (primary watchdog also stuck) |
| 245 | |
| 246 | def watchdog_timer() -> None: |
| 247 | time.sleep(seconds) |
| 248 | if _CANCEL_WATCHDOG.is_set(): |
| 249 | return |
| 250 | |
| 251 | warnings.warn(f"Watchdog timer expired after {seconds} seconds.") |
| 252 | |
| 253 | # Release any build locks first, before potentially-blocking diagnostics. |
| 254 | # If diagnostics hang, the deadman timer will still force-exit, but the |
| 255 | # lock will already be released so subsequent runs aren't blocked. |
| 256 | _release_held_build_locks() |
| 257 | |
| 258 | dump_thread_stacks() |
| 259 | ts_print(f"Watchdog timer expired after {seconds} seconds - forcing exit") |
| 260 | |
| 261 | # Dump outstanding running processes (if any) |
| 262 | try: |
| 263 | from ci.util.running_process_manager import ( |
| 264 | RunningProcessManagerSingleton, # noqa: PLC0415 |
| 265 | ) |
| 266 | |
| 267 | RunningProcessManagerSingleton.dump_active() |
| 268 | except KeyboardInterrupt as ki: |
| 269 | handle_keyboard_interrupt(ki) |
| 270 | raise |
| 271 | except Exception as e: |
| 272 | ts_print(f"Failed to dump active processes: {e}") |
| 273 | |
| 274 | import traceback # noqa: PLC0415 - lazy: only imported when watchdog fires |
| 275 | |
| 276 | traceback.print_stack() |
| 277 | time.sleep(0.5) |
| 278 | |
| 279 | os._exit(2) # Exit with error code 2 to indicate timeout (SIGTERM) |
| 280 | |
| 281 | thr = threading.Thread(target=watchdog_timer, daemon=True, name="WatchdogTimer") |
| 282 | thr.start() |
| 283 | |