Method run

scripts/benchmark/exp_utils.py:105–146

(self, exp: Experiment)

            all_gpus.update(gpus)

    def run(self, exp: Experiment):
        if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')):
            with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r', encoding='utf-8') as f:
                _json = json.load(f)
                if exp.eval_dataset and 'eval_result' not in _json['record']:
                    if not exp.do_eval:
                        logger.info(f'Experiment {exp.name} need eval, load from file.')
                        exp.load(_json)
                        exp.do_eval = True
                else:
                    logger.warn(f'Experiment {exp.name} already done, skip')
                    return

        if exp.do_eval:
            runtime = self._build_eval_cmd(exp)
            exp.runtime = runtime
            envs = deepcopy(runtime.get('env', {}))
            envs.update(os.environ)
            logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
            os.makedirs('exp', exist_ok=True)
            log_file = os.path.join('exp', f'{exp.name}.eval.log')
            exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
            self.exps.append(exp)
            self.assert_gpu_not_overlap()
            return

        if any([exp.name == e.name for e in self.exps]):
            raise ValueError(f'Why exp name duplicate? {exp.name}')
        elif exp.cmd == 'export' and any([exp.cmd == 'export' for exp in self.exps]):  # noqa
            raise AssertionError('Cannot run parallel export task.')
        else:
            exp.create_time = time.time()
            runtime = self._build_cmd(exp)
            exp.runtime = runtime
            envs = deepcopy(runtime.get('env', {}))
            envs.update(os.environ)
            logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
            os.makedirs('exp', exist_ok=True)
            log_file = os.path.join('exp', f'{exp.name}.{exp.cmd}.log')
            exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
            self.exps.append(exp)
            self.assert_gpu_not_overlap()

    def _build_eval_cmd(self, exp: Experiment):
        gpu = exp.eval_requirements.get('gpu', None)