Function proxy_worker_node_wait

autotest/utils/proxy_distributed_utils.py:154–187

Worker node waits by periodically checking if the master's proxy service is still alive. If the proxy becomes unreachable for several consecutive checks, assume master has finished. Args: manager: ProxyDistributedManager instance timeout_minutes: Maximum time to wait bef

(manager, timeout_minutes: int = 120)

def proxy_worker_node_wait(manager, timeout_minutes: int = 120):
    """Worker node waits by periodically checking if the master's proxy service
    is still alive. If the proxy becomes unreachable for several consecutive
    checks, assume master has finished.

    Args:
        manager: ProxyDistributedManager instance
        timeout_minutes: Maximum time to wait before giving up (default: 120 minutes)
    """
    print(f'⏸️ Worker node {manager.node_rank} entering monitoring mode...')

    max_checks = (timeout_minutes * 60) // WORKER_WAIT_INTERVAL
    consecutive_failures = 0
    max_consecutive_failures = 3

    for i in range(max_checks):
        if not is_port_open(manager.master_addr, manager.proxy_port, timeout=2.0):
            consecutive_failures += 1
            print(f'⚠️ Proxy connection to master failed ({consecutive_failures}/{max_consecutive_failures})')
            if consecutive_failures >= max_consecutive_failures:
                print('📡 Master proxy service stopped, worker node exiting')
                break
        else:
            consecutive_failures = 0

        if i % 4 == 0:
            elapsed = (i * WORKER_WAIT_INTERVAL) // 60
            print(f'⏳ Worker node {manager.node_rank} monitoring... Running for {elapsed} minutes')

        time.sleep(WORKER_WAIT_INTERVAL)
    else:
        print(f'⏰ Worker node {manager.node_rank} monitoring timed out ({timeout_minutes} minutes)')

    print(f'✅ Worker node {manager.node_rank} completed waiting')


class ProxyDistributedManager: