(model, max_seq_len, tasks, infer_path, ntrain=5, overwrite = False)
| 135 | return prompt |
| 136 | |
| 137 | def run_infer(model, max_seq_len, tasks, infer_path, ntrain=5, overwrite = False): |
| 138 | |
| 139 | for task in tasks: |
| 140 | |
| 141 | task_infer_path = os.path.join(infer_path, f'{task}_infer.jsonl') |
| 142 | if not overwrite and os.path.exists(task_infer_path): |
| 143 | print(f"{task_infer_path} existed, skip!") |
| 144 | continue |
| 145 | |
| 146 | print('Testing %s ...' % task) |
| 147 | dev_df = pd.read_csv(os.path.join(args.data_dir, "data/dev", task + "_dev.csv"), header=None)[:args.ntrain] |
| 148 | test_df = pd.read_csv(os.path.join(args.data_dir, "data/test", task + "_test.csv"), header=None) |
| 149 | few_shot_prompt = generate_few_shot_prompt(dev_df, task, ntrain) |
| 150 | |
| 151 | test_set = [] |
| 152 | answer_set = [] |
| 153 | for i in range(test_df.shape[0]): |
| 154 | prompt = format_example(test_df, i, include_answer=False) |
| 155 | full_prompt = resize_prompt( |
| 156 | model.tokenizer, |
| 157 | max_seq_len, |
| 158 | few_shot_prompt+prompt |
| 159 | ) |
| 160 | |
| 161 | test_set.append(full_prompt) |
| 162 | target_ans = test_df.iloc[i, test_df.shape[1]-1] |
| 163 | answer_set.append(target_ans) |
| 164 | |
| 165 | batch_prompt = batch_data(test_set, batch_size=8) |
| 166 | |
| 167 | res_completions = [] |
| 168 | for batch_input in tqdm(batch_prompt): |
| 169 | |
| 170 | outputs = model.generate(prompts=batch_input, images=None, max_gen_len=1) |
| 171 | |
| 172 | for output in outputs: |
| 173 | res_completions.append(output) |
| 174 | |
| 175 | torch.distributed.barrier() |
| 176 | if torch.distributed.get_rank() == 0: |
| 177 | |
| 178 | with jsonlines.open(task_infer_path, mode='w') as writer: |
| 179 | for (completion, prompt_answer) in zip(res_completions, answer_set): |
| 180 | record = { |
| 181 | 'completion': completion, |
| 182 | 'target_ans': prompt_answer |
| 183 | } |
| 184 | writer.write(record) |
| 185 | writer.close() |
| 186 | |
| 187 | def run_eval(tasks, infer_path): |
| 188 |
no test coverage detected