| 169 | |
| 170 | |
| 171 | def validate_servers(args): |
| 172 | probe = {"conversations": [{"role": "user", "content": "Hello"}]} |
| 173 | server_results = {} |
| 174 | server_count = len(args.server_address) |
| 175 | |
| 176 | print(f"Validating {server_count} sglang servers in parallel...", flush=True) |
| 177 | with ThreadPoolExecutor(max_workers=server_count) as executor: |
| 178 | future_to_server = { |
| 179 | executor.submit( |
| 180 | validate_server, |
| 181 | args, |
| 182 | server_address, |
| 183 | probe, |
| 184 | ): server_address |
| 185 | for server_address in args.server_address |
| 186 | } |
| 187 | |
| 188 | for completed_count, future in enumerate(as_completed(future_to_server), 1): |
| 189 | server_address = future_to_server[future] |
| 190 | try: |
| 191 | _, result, elapsed = future.result() |
| 192 | except Exception as exc: |
| 193 | result = {"status": "error", "error": str(exc)} |
| 194 | elapsed = 0.0 |
| 195 | |
| 196 | if result.get("status") == "success": |
| 197 | server_results[server_address] = True |
| 198 | print( |
| 199 | f"[validate {completed_count}/{server_count}] " |
| 200 | f"ok server {server_address} elapsed={elapsed:.2f}s", |
| 201 | flush=True, |
| 202 | ) |
| 203 | else: |
| 204 | server_results[server_address] = False |
| 205 | print( |
| 206 | f"[validate {completed_count}/{server_count}] " |
| 207 | f"skip server {server_address} elapsed={elapsed:.2f}s: " |
| 208 | f"{result.get('error')}", |
| 209 | flush=True, |
| 210 | ) |
| 211 | |
| 212 | valid_servers = [ |
| 213 | server_address |
| 214 | for server_address in args.server_address |
| 215 | if server_results[server_address] |
| 216 | ] |
| 217 | invalid_servers = [ |
| 218 | server_address |
| 219 | for server_address in args.server_address |
| 220 | if not server_results[server_address] |
| 221 | ] |
| 222 | |
| 223 | print(f"Available servers ({len(valid_servers)}/{server_count}): {valid_servers}") |
| 224 | print( |
| 225 | f"Unavailable servers ({len(invalid_servers)}/{server_count}): " |
| 226 | f"{invalid_servers}" |
| 227 | ) |
| 228 | if not valid_servers: |