(exp: Experiment)
| 271 | |
| 272 | @staticmethod |
| 273 | def _get_metric(exp: Experiment): |
| 274 | if exp.do_eval: |
| 275 | if os.path.isfile(os.path.join('exp', f'{exp.name}.eval.log')): |
| 276 | with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r', encoding='utf-8') as f: |
| 277 | for line in f.readlines(): |
| 278 | if 'Final report:' in line: |
| 279 | return json.loads(line.split('Final report:')[1].replace('\'', '"')) |
| 280 | elif exp.cmd == 'export': |
| 281 | exp_args = ExportArguments(**exp.args) |
| 282 | if exp_args.quant_bits > 0: |
| 283 | if exp_args.ckpt_dir is None: |
| 284 | path = f'{exp_args.model_type}-{exp_args.quant_method}-int{exp_args.quant_bits}' |
| 285 | else: |
| 286 | ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir) |
| 287 | path = os.path.join(ckpt_dir, f'{ckpt_name}-{exp_args.quant_method}-int{exp_args.quant_bits}') |
| 288 | else: |
| 289 | ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir) |
| 290 | path = os.path.join(ckpt_dir, f'{ckpt_name}-merged') |
| 291 | if os.path.exists(path): |
| 292 | shutil.rmtree(exp.name, ignore_errors=True) |
| 293 | os.makedirs(exp.name, exist_ok=True) |
| 294 | shutil.move(path, os.path.join(exp.name, path)) |
| 295 | return { |
| 296 | 'best_model_checkpoint': os.path.join(exp.name, path), |
| 297 | } |
| 298 | else: |
| 299 | logging_dir = exp.runtime.get('logging_dir') |
| 300 | logging_file = os.path.join(logging_dir, '..', 'logging.jsonl') |
| 301 | if os.path.isfile(logging_file): |
| 302 | with open(logging_file, 'r', encoding='utf-8') as f: |
| 303 | for line in f.readlines(): |
| 304 | if 'model_info' in line: |
| 305 | return json.loads(line) |
| 306 | return None |
| 307 | |
| 308 | @staticmethod |
| 309 | def write_record(exp: Experiment): |
no test coverage detected