(args, model, tokenizer, prefix="")
| 219 | |
| 220 | |
| 221 | def evaluate(args, model, tokenizer, prefix=""): |
| 222 | dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True) |
| 223 | |
| 224 | if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: |
| 225 | os.makedirs(args.output_dir) |
| 226 | |
| 227 | args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) |
| 228 | # Note that DistributedSampler samples randomly |
| 229 | eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) |
| 230 | eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) |
| 231 | |
| 232 | # Eval! |
| 233 | logger.info("***** Running evaluation {} *****".format(prefix)) |
| 234 | logger.info(" Num examples = %d", len(dataset)) |
| 235 | logger.info(" Batch size = %d", args.eval_batch_size) |
| 236 | all_results = [] |
| 237 | for batch in tqdm(eval_dataloader, desc="Evaluating"): |
| 238 | model.eval() |
| 239 | batch = tuple(t.to(args.device) for t in batch) |
| 240 | with torch.no_grad(): |
| 241 | inputs = {"input_ids": batch[0], "attention_mask": batch[1]} |
| 242 | if args.model_type != "distilbert": |
| 243 | inputs["token_type_ids"] = None if args.model_type == "xlm" else batch[2] # XLM don't use segment_ids |
| 244 | example_indices = batch[3] |
| 245 | if args.model_type in ["xlnet", "xlm"]: |
| 246 | inputs.update({"cls_index": batch[4], "p_mask": batch[5]}) |
| 247 | outputs = model(**inputs) |
| 248 | |
| 249 | for i, example_index in enumerate(example_indices): |
| 250 | eval_feature = features[example_index.item()] |
| 251 | unique_id = int(eval_feature.unique_id) |
| 252 | if args.model_type in ["xlnet", "xlm"]: |
| 253 | # XLNet uses a more complex post-processing procedure |
| 254 | result = RawResultExtended( |
| 255 | unique_id=unique_id, |
| 256 | start_top_log_probs=to_list(outputs[0][i]), |
| 257 | start_top_index=to_list(outputs[1][i]), |
| 258 | end_top_log_probs=to_list(outputs[2][i]), |
| 259 | end_top_index=to_list(outputs[3][i]), |
| 260 | cls_logits=to_list(outputs[4][i]), |
| 261 | ) |
| 262 | else: |
| 263 | result = RawResult( |
| 264 | unique_id=unique_id, start_logits=to_list(outputs[0][i]), end_logits=to_list(outputs[1][i]) |
| 265 | ) |
| 266 | all_results.append(result) |
| 267 | |
| 268 | # Compute predictions |
| 269 | output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix)) |
| 270 | output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix)) |
| 271 | if args.version_2_with_negative: |
| 272 | output_null_log_odds_file = os.path.join(args.output_dir, "null_odds_{}.json".format(prefix)) |
| 273 | else: |
| 274 | output_null_log_odds_file = None |
| 275 | |
| 276 | if args.model_type in ["xlnet", "xlm"]: |
| 277 | # XLNet uses a more complex post-processing procedure |
| 278 | write_predictions_extended( |
no test coverage detected