()
| 125 | |
| 126 | |
| 127 | def main(): |
| 128 | args = parse_args() |
| 129 | # initialize logger |
| 130 | logger = get_logger(log_level='INFO') |
| 131 | cfg = Config.fromfile(args.config) |
| 132 | cfg.setdefault('work_dir', './outputs/default/') |
| 133 | |
| 134 | assert args.reuse, 'Please provide the experienment work dir.' |
| 135 | if args.reuse: |
| 136 | if args.reuse == 'latest': |
| 137 | if not os.path.exists(cfg.work_dir) or not os.listdir( |
| 138 | cfg.work_dir): |
| 139 | logger.warning('No previous results to reuse!') |
| 140 | else: |
| 141 | dirs = os.listdir(cfg.work_dir) |
| 142 | dir_time_str = sorted(dirs)[-1] |
| 143 | else: |
| 144 | dir_time_str = args.reuse |
| 145 | logger.info(f'Reusing experiements from {dir_time_str}') |
| 146 | # update "actual" work_dir |
| 147 | cfg['work_dir'] = osp.join(cfg.work_dir, dir_time_str) |
| 148 | |
| 149 | for model in cfg.models: |
| 150 | model_abbr = model_abbr_from_cfg(model) |
| 151 | for dataset in cfg.datasets: |
| 152 | dataset_abbr = dataset_abbr_from_cfg(dataset) |
| 153 | filename = get_infer_output_path( |
| 154 | model, dataset, osp.join(cfg.work_dir, 'predictions')) |
| 155 | |
| 156 | succeed, ori_prompt_strs, pred_strs = collect_preds(filename) |
| 157 | if not succeed: |
| 158 | continue |
| 159 | |
| 160 | # infer the language type |
| 161 | for k, v in _LANGUAGE_NAME_DICT.items(): |
| 162 | if k in dataset_abbr: |
| 163 | lang = k |
| 164 | task = v |
| 165 | break |
| 166 | |
| 167 | # special postprocess for GPT |
| 168 | if model_abbr in [ |
| 169 | 'WizardCoder-1B-V1.0', |
| 170 | 'WizardCoder-3B-V1.0', |
| 171 | 'WizardCoder-15B-V1.0', |
| 172 | 'WizardCoder-Python-13B-V1.0', |
| 173 | 'WizardCoder-Python-34B-V1.0', |
| 174 | ]: |
| 175 | predictions = [{ |
| 176 | 'task_id': f'{task}/{i}', |
| 177 | 'generation': wizardcoder_postprocess(pred), |
| 178 | } for i, pred in enumerate(pred_strs)] |
| 179 | elif 'CodeLlama' not in model_abbr and lang == 'python': |
| 180 | predictions = [{ |
| 181 | 'task_id': |
| 182 | f'{task}/{i}', |
| 183 | 'generation': |
| 184 | gpt_python_postprocess(ori_prompt, pred), |
no test coverage detected