Method prepare_experiments

scripts/benchmark/exp_utils.py:236–270

(self, args: Any)

        return list(free_gpu)[:n]

    def prepare_experiments(self, args: Any):
        experiments = []
        for config_file in args.config:
            with open(config_file, 'r', encoding='utf-8') as f:
                group = os.path.basename(config_file)
                group = group[:-5]
                content = json.load(f)
                exps = content['experiment']
                for exp in exps:
                    main_cfg = deepcopy(content)
                    name = exp['name']
                    cmd = main_cfg['cmd']
                    run_args = main_cfg['args']
                    env = main_cfg.get('env', {})
                    requirements = main_cfg.get('requirements', {})
                    eval_requirements = main_cfg.get('eval_requirements', {})
                    eval_dataset = main_cfg.get('eval_dataset', {})
                    if 'args' in exp:
                        run_args.update(exp['args'])
                    if 'requirements' in exp:
                        requirements.update(exp['requirements'])
                    if 'env' in exp:
                        env.update(exp['env'])
                    experiments.append(
                        Experiment(
                            group=group,
                            name=name,
                            cmd=cmd,
                            args=run_args,
                            env=env,
                            requirements=requirements,
                            eval_requirements=eval_requirements,
                            eval_dataset=eval_dataset,
                            input_args=args))
        return experiments

    @staticmethod
    def _get_metric(exp: Experiment):