(self)
| 235 | self.device = device |
| 236 | |
| 237 | def train(self): |
| 238 | cal_metric_during_train = self.cfg['Global'].get( |
| 239 | 'cal_metric_during_train', False) |
| 240 | log_smooth_window = self.cfg['Global']['log_smooth_window'] |
| 241 | epoch_num = self.cfg['Global']['epoch_num'] |
| 242 | print_batch_step = self.cfg['Global']['print_batch_step'] |
| 243 | eval_epoch_step = self.cfg['Global'].get('eval_epoch_step', 1) |
| 244 | |
| 245 | start_eval_epoch = 0 |
| 246 | if self.valid_dataloader is not None: |
| 247 | if type(eval_epoch_step) == list and len(eval_epoch_step) >= 2: |
| 248 | start_eval_epoch = eval_epoch_step[0] |
| 249 | eval_epoch_step = eval_epoch_step[1] |
| 250 | if len(self.valid_dataloader) == 0: |
| 251 | start_eval_epoch = 1e111 |
| 252 | self.logger.info( |
| 253 | 'No Images in eval dataset, evaluation during training will be disabled' |
| 254 | ) |
| 255 | self.logger.info( |
| 256 | f'During the training process, after the {start_eval_epoch}th epoch, ' |
| 257 | f'an evaluation is run every {eval_epoch_step} epoch') |
| 258 | else: |
| 259 | start_eval_epoch = 1e111 |
| 260 | |
| 261 | eval_batch_step = self.cfg['Global']['eval_batch_step'] |
| 262 | |
| 263 | global_step = self.status.get('global_step', 0) |
| 264 | |
| 265 | start_eval_step = 0 |
| 266 | if type(eval_batch_step) == list and len(eval_batch_step) >= 2: |
| 267 | start_eval_step = eval_batch_step[0] |
| 268 | eval_batch_step = eval_batch_step[1] |
| 269 | if self.valid_dataloader is not None and len(self.valid_dataloader) == 0: |
| 270 | self.logger.info( |
| 271 | 'No Images in eval dataset, evaluation during training ' |
| 272 | 'will be disabled') |
| 273 | start_eval_step = 1e111 |
| 274 | self.logger.info( |
| 275 | 'During the training process, after the {}th iteration, ' |
| 276 | 'an evaluation is run every {} iterations'.format( |
| 277 | start_eval_step, eval_batch_step)) |
| 278 | |
| 279 | save_epoch_step = self.cfg['Global'].get('save_epoch_step', [0, 1]) |
| 280 | start_save_epoch = save_epoch_step[0] |
| 281 | save_epoch_step = save_epoch_step[1] |
| 282 | |
| 283 | start_epoch = self.status.get('epoch', 1) |
| 284 | self.best_metric = self.status.get('metrics', {}) |
| 285 | if self.eval_class.main_indicator not in self.best_metric: |
| 286 | self.best_metric[self.eval_class.main_indicator] = 0 |
| 287 | train_stats = TrainingStats(log_smooth_window, ['lr']) |
| 288 | self.model.train() |
| 289 | |
| 290 | total_samples = 0 |
| 291 | train_reader_cost = 0.0 |
| 292 | train_batch_cost = 0.0 |
| 293 | reader_start = time.time() |
| 294 | eta_meter = AverageMeter() |
no test coverage detected