Handle a hung test by dumping stack trace and killing it. Args: pid: Process ID of hung test test_name: Name of the test timeout_seconds: How long the test has been running Returns: Formatted error message with stack trace
(pid: int, test_name: str, timeout_seconds: float)
| 231 | |
| 232 | |
| 233 | def handle_hung_test(pid: int, test_name: str, timeout_seconds: float) -> str: |
| 234 | """ |
| 235 | Handle a hung test by dumping stack trace and killing it. |
| 236 | |
| 237 | Args: |
| 238 | pid: Process ID of hung test |
| 239 | test_name: Name of the test |
| 240 | timeout_seconds: How long the test has been running |
| 241 | |
| 242 | Returns: |
| 243 | Formatted error message with stack trace |
| 244 | """ |
| 245 | ts_print("=" * 80) |
| 246 | ts_print(f"🚨 HUNG TEST DETECTED: {test_name}") |
| 247 | ts_print(f"⏱️ Test exceeded timeout of {timeout_seconds:.1f} seconds") |
| 248 | ts_print(f"🔍 Process ID: {pid}") |
| 249 | ts_print("=" * 80) |
| 250 | |
| 251 | # Dump stack trace |
| 252 | stack_trace = dump_stack_trace_lldb(pid, test_name) |
| 253 | |
| 254 | # Format output |
| 255 | error_msg = f"\n{'=' * 80}\n" |
| 256 | error_msg += f"HUNG TEST: {test_name}\n" |
| 257 | error_msg += f"Timeout: {timeout_seconds:.1f}s\n" |
| 258 | error_msg += f"PID: {pid}\n" |
| 259 | error_msg += "=" * 80 + "\n" |
| 260 | |
| 261 | if stack_trace: |
| 262 | error_msg += "\nTHREAD STACK TRACES:\n" |
| 263 | error_msg += "-" * 80 + "\n" |
| 264 | error_msg += stack_trace |
| 265 | error_msg += "\n" + "-" * 80 + "\n" |
| 266 | else: |
| 267 | error_msg += "\n⚠️ Could not obtain stack trace (debugger not available)\n" |
| 268 | |
| 269 | error_msg += "=" * 80 + "\n" |
| 270 | |
| 271 | # Print to console |
| 272 | ts_print(error_msg) |
| 273 | |
| 274 | # Kill the hung process |
| 275 | try: |
| 276 | if platform.system() == "Windows": |
| 277 | RunningProcess.run( |
| 278 | ["taskkill", "/F", "/PID", str(pid)], cwd=None, check=False, timeout=5 |
| 279 | ) |
| 280 | else: |
| 281 | RunningProcess.run( |
| 282 | ["kill", "-9", str(pid)], cwd=None, check=False, timeout=5 |
| 283 | ) |
| 284 | ts_print(f"🔪 Killed hung process (PID {pid})") |
| 285 | except KeyboardInterrupt as ki: |
| 286 | handle_keyboard_interrupt(ki) |
| 287 | except Exception as e: |
| 288 | ts_print(f"⚠️ Failed to kill process: {e}") |
| 289 | |
| 290 | return error_msg |