Get the config object given args. Only a few argument combinations are accepted (priority from high to low) 1. args.config 2. args.models and args.datasets 3. Huggingface parameter groups and args.datasets
(args)
| 85 | |
| 86 | |
| 87 | def get_config_from_arg(args) -> Config: |
| 88 | """Get the config object given args. |
| 89 | |
| 90 | Only a few argument combinations are accepted (priority from high to low) |
| 91 | 1. args.config |
| 92 | 2. args.models and args.datasets |
| 93 | 3. Huggingface parameter groups and args.datasets |
| 94 | """ |
| 95 | |
| 96 | if args.config: |
| 97 | config = Config.fromfile(args.config, format_python_code=False) |
| 98 | config = try_fill_in_custom_cfgs(config) |
| 99 | # set infer accelerator if needed |
| 100 | if args.accelerator in ['vllm', 'lmdeploy']: |
| 101 | config['models'] = change_accelerator(config['models'], args.accelerator) |
| 102 | if config.get('eval', {}).get('partitioner', {}).get('models') is not None: |
| 103 | config['eval']['partitioner']['models'] = change_accelerator(config['eval']['partitioner']['models'], args.accelerator) |
| 104 | if config.get('eval', {}).get('partitioner', {}).get('base_models') is not None: |
| 105 | config['eval']['partitioner']['base_models'] = change_accelerator(config['eval']['partitioner']['base_models'], args.accelerator) |
| 106 | if config.get('eval', {}).get('partitioner', {}).get('compare_models') is not None: |
| 107 | config['eval']['partitioner']['compare_models'] = change_accelerator(config['eval']['partitioner']['compare_models'], args.accelerator) |
| 108 | if config.get('eval', {}).get('partitioner', {}).get('judge_models') is not None: |
| 109 | config['eval']['partitioner']['judge_models'] = change_accelerator(config['eval']['partitioner']['judge_models'], args.accelerator) |
| 110 | if config.get('judge_models') is not None: |
| 111 | config['judge_models'] = change_accelerator(config['judge_models'], args.accelerator) |
| 112 | return config |
| 113 | |
| 114 | # parse dataset args |
| 115 | if not args.datasets and not args.custom_dataset_path: |
| 116 | raise ValueError('You must specify "--datasets" or "--custom-dataset-path" if you do not specify a config file path.') |
| 117 | datasets = [] |
| 118 | if args.datasets: |
| 119 | script_dir = os.path.dirname(os.path.abspath(__file__)) |
| 120 | parent_dir = os.path.dirname(script_dir) |
| 121 | default_configs_dir = os.path.join(parent_dir, 'configs') |
| 122 | datasets_dir = [ |
| 123 | os.path.join(args.config_dir, 'datasets'), |
| 124 | os.path.join(args.config_dir, 'dataset_collections'), |
| 125 | os.path.join(default_configs_dir, './datasets'), |
| 126 | os.path.join(default_configs_dir, './dataset_collections') |
| 127 | |
| 128 | ] |
| 129 | for dataset_arg in args.datasets: |
| 130 | if '/' in dataset_arg: |
| 131 | dataset_name, dataset_suffix = dataset_arg.split('/', 1) |
| 132 | dataset_key_suffix = dataset_suffix |
| 133 | else: |
| 134 | dataset_name = dataset_arg |
| 135 | dataset_key_suffix = '_datasets' |
| 136 | |
| 137 | for dataset in match_cfg_file(datasets_dir, [dataset_name]): |
| 138 | logger.info(f'Loading {dataset[0]}: {dataset[1]}') |
| 139 | cfg = Config.fromfile(dataset[1]) |
| 140 | for k in cfg.keys(): |
| 141 | if k.endswith(dataset_key_suffix): |
| 142 | datasets += cfg[k] |
| 143 | else: |
| 144 | dataset = {'path': args.custom_dataset_path} |
no test coverage detected