(args, model, tokenizer, prefix="")
| 234 | |
| 235 | |
| 236 | def evaluate(args, model, tokenizer, prefix=""): |
| 237 | eval_task_names = (args.task_name,) |
| 238 | eval_outputs_dirs = (args.output_dir,) |
| 239 | |
| 240 | results = {} |
| 241 | for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): |
| 242 | eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) |
| 243 | |
| 244 | if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: |
| 245 | os.makedirs(eval_output_dir) |
| 246 | |
| 247 | args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) |
| 248 | # Note that DistributedSampler samples randomly |
| 249 | eval_sampler = SequentialSampler(eval_dataset) |
| 250 | eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) |
| 251 | |
| 252 | # multi-gpu eval |
| 253 | if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): |
| 254 | model = torch.nn.DataParallel(model) |
| 255 | |
| 256 | # Eval! |
| 257 | logger.info("***** Running evaluation {} *****".format(prefix)) |
| 258 | logger.info(" Num examples = %d", len(eval_dataset)) |
| 259 | logger.info(" Batch size = %d", args.eval_batch_size) |
| 260 | eval_loss = 0.0 |
| 261 | nb_eval_steps = 0 |
| 262 | preds = None |
| 263 | out_label_ids = None |
| 264 | for batch in tqdm(eval_dataloader, desc="Evaluating"): |
| 265 | model.eval() |
| 266 | batch = tuple(t.to(args.device) for t in batch) |
| 267 | |
| 268 | with torch.no_grad(): |
| 269 | inputs = {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]} |
| 270 | if args.model_type != "distilbert": |
| 271 | inputs["token_type_ids"] = ( |
| 272 | batch[2] if args.model_type in ["bert"] else None |
| 273 | ) # XLM and DistilBERT don't use segment_ids |
| 274 | outputs = model(**inputs) |
| 275 | tmp_eval_loss, logits = outputs[:2] |
| 276 | |
| 277 | eval_loss += tmp_eval_loss.mean().item() |
| 278 | nb_eval_steps += 1 |
| 279 | if preds is None: |
| 280 | preds = logits.detach().cpu().numpy() |
| 281 | out_label_ids = inputs["labels"].detach().cpu().numpy() |
| 282 | else: |
| 283 | preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) |
| 284 | out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0) |
| 285 | |
| 286 | eval_loss = eval_loss / nb_eval_steps |
| 287 | if args.output_mode == "classification": |
| 288 | preds = np.argmax(preds, axis=1) |
| 289 | else: |
| 290 | raise ValueError("No other `output_mode` for XNLI.") |
| 291 | result = compute_metrics(eval_task, preds, out_label_ids) |
| 292 | results.update(result) |
| 293 |
no test coverage detected