| 201 | |
| 202 | |
| 203 | def eval_test(model_path, |
| 204 | eval_path, |
| 205 | case_name, |
| 206 | port=DEFAULT_PORT, |
| 207 | test_type='infer', |
| 208 | extra_config=None, |
| 209 | eval_config_name='default', |
| 210 | **kwargs): |
| 211 | if extra_config is None: |
| 212 | extra_config = {} |
| 213 | work_dir = None |
| 214 | try: |
| 215 | |
| 216 | work_dir = os.path.join(eval_path, f'wk_{case_name}') |
| 217 | timestamp = time.strftime('%Y%m%d_%H%M%S') |
| 218 | eval_log = os.path.join(eval_path, f'log_{case_name}_{test_type}_{timestamp}.log') |
| 219 | temp_config_path = os.path.join(eval_path, f'temp_{case_name}.py') |
| 220 | |
| 221 | current_dir = os.path.dirname(os.path.abspath(__file__)) |
| 222 | parent_dir = os.path.dirname(current_dir) |
| 223 | |
| 224 | if eval_config_name == 'longtext-512k': |
| 225 | config_file = os.path.join(parent_dir, 'evaluate/eval_config_chat_512_longtext.py') |
| 226 | elif eval_config_name == 'longtext-256k': |
| 227 | config_file = os.path.join(parent_dir, 'evaluate/eval_config_chat_longtext.py') |
| 228 | else: |
| 229 | config_file = os.path.join(parent_dir, 'evaluate/eval_config_chat.py') |
| 230 | |
| 231 | print(f'Starting OpenCompass evaluation for model: {model_path}') |
| 232 | print(f'Model path: {model_path}') |
| 233 | print(f'Case: {case_name}') |
| 234 | print(f'Config file: {config_file}') |
| 235 | |
| 236 | original_cwd = os.getcwd() |
| 237 | os.makedirs(work_dir, exist_ok=True) |
| 238 | |
| 239 | test_url = f'http://{DEFAULT_SERVER}:{port}/v1' |
| 240 | |
| 241 | try: |
| 242 | if test_type == 'infer': |
| 243 | if not os.path.exists(config_file): |
| 244 | return False, f'Config file {config_file} not found' |
| 245 | |
| 246 | with _mmengine_lazy_allow_lazyattr_call(): |
| 247 | cfg = Config.fromfile(config_file) |
| 248 | |
| 249 | cfg.MODEL_NAME = case_name |
| 250 | cfg.MODEL_PATH = model_path |
| 251 | cfg.API_BASE = test_url # noqa: E231 |
| 252 | |
| 253 | if cfg.models and len(cfg.models) > 0: |
| 254 | model_cfg = cfg.models[0] |
| 255 | model_cfg['abbr'] = case_name |
| 256 | model_cfg['path'] = case_name |
| 257 | model_cfg['openai_api_base'] = test_url |
| 258 | model_cfg['tokenizer_path'] = model_path |
| 259 | |
| 260 | for key, value in kwargs.items(): |