| 185 | |
| 186 | |
| 187 | def predict_data( |
| 188 | input_path: str, output_path: str, test: int, |
| 189 | device, gpt: str, batch_size: int |
| 190 | ) -> Dataset: |
| 191 | if 'mix' in input_path: |
| 192 | data = list() |
| 193 | for mode in ('text', 'sent'): |
| 194 | data.append(Dataset.from_json(output_path.replace('mix', mode))) |
| 195 | return concatenate_datasets(data) |
| 196 | |
| 197 | dataset = read_data(input_path) |
| 198 | tokenizer = GPT2Tokenizer.from_pretrained(gpt) |
| 199 | tokenizer.pad_token = tokenizer.eos_token |
| 200 | model = GPT2LMHeadModel.from_pretrained(gpt).to(device) |
| 201 | model.eval() |
| 202 | kwargs = dict(model=model, tokenizer=tokenizer, device=device) |
| 203 | |
| 204 | if test > 0: |
| 205 | processor = partial(gltr_batched, **kwargs) |
| 206 | else: |
| 207 | processor = partial(ppl_batched, **kwargs) |
| 208 | |
| 209 | with torch.no_grad(): |
| 210 | dataset= dataset.map( |
| 211 | processor, batched=True, batch_size=batch_size, desc='running gpt2' |
| 212 | ) |
| 213 | dataset.to_json(output_path, orient='records', lines=True, force_ascii=False) |
| 214 | printf(output_path) |
| 215 | return dataset |
| 216 | |
| 217 | |
| 218 | def compute_metrics(preds, y_true, y_scores): |