Function run_eval_test

autotest/evaluate/test_api_evaluate.py:112–186

Run test with specified evaluation configuration.

(config, run_config, worker_id, test_type='infer', eval_config_name='default', eval_subpath=None)

def run_eval_test(config, run_config, worker_id, test_type='infer', eval_config_name='default', eval_subpath=None):
    """Run test with specified evaluation configuration."""
    eval_config_name = resolve_eval_config_name(config, run_config, eval_config_name)
    preset_config = get_eval_preset_config(config, run_config, eval_config_name)
    eval_path = config.get('eval_path')
    if eval_subpath:
        eval_path = os.path.join(eval_path, eval_subpath)
        os.makedirs(eval_path, exist_ok=True)

    total_gpus = int(os.environ.get('TOTAL_GPU_COUNT', '8'))
    work_num = int(total_gpus / run_config.get('parallel_config', {}).get('tp', 1))

    extra_config = {'max-num-workers': min(work_num * 16, 64)}

    case_name = get_case_str_by_config(run_config)

    if test_type == 'infer':
        proxy_pid, proxy_process = start_proxy_server(config.get('server_log_path'), constant.PROXY_PORT,
                                                      f'{case_name}_infer')
        run_config_new = run_config.copy()
        if 'extra_params' not in run_config_new:
            run_config_new['extra_params'] = {}
        run_config_new['extra_params']['proxy-url'] = f'http://{constant.DEFAULT_SERVER}:{constant.PROXY_PORT}'
        run_config_new['extra_params']['server-name'] = constant.DEFAULT_SERVER

        from concurrent.futures import ThreadPoolExecutor

        def run_openai_service_start(i):
            return start_openai_service(config, run_config_new, f'gw{i}')

        with ThreadPoolExecutor(max_workers=work_num) as executor:
            futures = [executor.submit(run_openai_service_start, i) for i in range(int(work_num))]
        results = []
        for future in futures:
            pid, content = future.result()
            results.append((pid, content))

        try:
            model_path = os.path.join(config.get('model_path'), run_config.get('model'))
            eval_test(model_path,
                      eval_path,
                      case_name,
                      port=constant.PROXY_PORT,
                      test_type=test_type,
                      extra_config=extra_config,
                      eval_config_name=eval_config_name,
                      **preset_config)
        finally:
            for i in range(work_num):
                terminate_restful_api(f'gw{i}')
            stop_restful_api(proxy_pid, proxy_process)
    else:  # eval
        port = constant.PROXY_PORT + get_workerid(worker_id)
        proxy_pid, proxy_process = start_proxy_server(config.get('server_log_path'), port, f'{case_name}_eval')
        eval_run_config = build_eval_judge_run_config(
            config, f'http://{constant.DEFAULT_SERVER}:{port}')

        pid, content = start_openai_service(config, eval_run_config, worker_id)