(self)
| 259 | logger.info("[EvalCallback] Will evaluate every {} epochs".format(eval_period)) |
| 260 | |
| 261 | def _eval(self): |
| 262 | logdir = self._output_dir |
| 263 | if cfg.TRAINER == 'replicated': |
| 264 | all_results = multithread_predict_dataflow(self.dataflows, self.predictors) |
| 265 | else: |
| 266 | filenames = [os.path.join( |
| 267 | logdir, 'outputs{}-part{}.json'.format(self.global_step, rank) |
| 268 | ) for rank in range(hvd.local_size())] |
| 269 | |
| 270 | if self._horovod_run_eval: |
| 271 | local_results = predict_dataflow(self.dataflow, self.predictor) |
| 272 | fname = filenames[hvd.local_rank()] |
| 273 | with open(fname, 'w') as f: |
| 274 | json.dump(local_results, f) |
| 275 | self.barrier.eval() |
| 276 | if hvd.rank() > 0: |
| 277 | return |
| 278 | all_results = [] |
| 279 | for fname in filenames: |
| 280 | with open(fname, 'r') as f: |
| 281 | obj = json.load(f) |
| 282 | all_results.extend(obj) |
| 283 | os.unlink(fname) |
| 284 | |
| 285 | scores = DatasetRegistry.get(self._eval_dataset).eval_inference_results(all_results) |
| 286 | for k, v in scores.items(): |
| 287 | self.trainer.monitors.put_scalar(self._eval_dataset + '-' + k, v) |
| 288 | |
| 289 | def _trigger_epoch(self): |
| 290 | if self.epoch_num in self.epochs_to_eval: |
no test coverage detected