(config,
run_config,
worker_id,
test_type='infer',
manager=None,
eval_config_name='default',
eval_subpath=None)
| 58 | |
| 59 | |
| 60 | def _run_proxy_distributed_test(config, |
| 61 | run_config, |
| 62 | worker_id, |
| 63 | test_type='infer', |
| 64 | manager=None, |
| 65 | eval_config_name='default', |
| 66 | eval_subpath=None): |
| 67 | assert manager is not None, 'Manager instance must be provided' |
| 68 | |
| 69 | if eval_subpath is None: |
| 70 | eval_config_name = resolve_eval_config_name(config, run_config, eval_config_name) |
| 71 | |
| 72 | preset_config = get_eval_preset_config(config, run_config, eval_config_name) |
| 73 | model_name = run_config['model'] |
| 74 | model_path = os.path.join(config['model_path'], model_name) |
| 75 | |
| 76 | api_server = ApiServerPerTest(proxy_manager=manager, config=config, run_config=run_config) |
| 77 | api_server.start() |
| 78 | |
| 79 | try: |
| 80 | if manager.is_master: |
| 81 | api_server.wait_until_ready() |
| 82 | print(f'🧪 Master node executing {test_type} test ({eval_config_name})...') |
| 83 | eval_path = config.get('eval_path') |
| 84 | if eval_subpath: |
| 85 | eval_path = os.path.join(eval_path, eval_subpath) |
| 86 | os.makedirs(eval_path, exist_ok=True) |
| 87 | case_name = get_case_str_by_config(run_config) |
| 88 | |
| 89 | extra_config = {'max-num-workers': 16} |
| 90 | |
| 91 | result, msg = eval_test(model_path, |
| 92 | eval_path, |
| 93 | case_name, |
| 94 | port=constant.PROXY_PORT, |
| 95 | test_type=test_type, |
| 96 | extra_config=extra_config, |
| 97 | eval_config_name=eval_config_name, |
| 98 | **preset_config) |
| 99 | assert result, f'❌ {test_type} test failed: {msg}' |
| 100 | print(f'✅ {test_type} test passed') |
| 101 | |
| 102 | else: |
| 103 | print(f'⏸️ Worker node {manager.node_rank} waiting for master to complete test...') |
| 104 | proxy_worker_node_wait(manager, timeout_minutes=4880) |
| 105 | |
| 106 | finally: |
| 107 | api_server.cleanup() |
| 108 | if manager.is_master: |
| 109 | time.sleep(1) |
| 110 | |
| 111 | |
| 112 | def run_eval_test(config, run_config, worker_id, test_type='infer', eval_config_name='default', eval_subpath=None): |
no test coverage detected