(self)
| 1311 | return output |
| 1312 | |
| 1313 | def train(self): |
| 1314 | model = self.get_model() |
| 1315 | # run all epochs |
| 1316 | for epoch in range(self.current_epoch, 1000000): |
| 1317 | # set seed for distributed sampler (enables shuffling for each epoch) |
| 1318 | if self.use_ddp and hasattr(self.get_train_dataloader().sampler, 'set_epoch'): |
| 1319 | self.get_train_dataloader().sampler.set_epoch(epoch) |
| 1320 | |
| 1321 | # get model |
| 1322 | model = self.get_model() |
| 1323 | |
| 1324 | # update training progress in trainer and model |
| 1325 | model.current_epoch = epoch |
| 1326 | self.current_epoch = epoch |
| 1327 | |
| 1328 | total_val_batches = 0 |
| 1329 | if not self.disable_validation: |
| 1330 | # val can be checked multiple times in epoch |
| 1331 | is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0 |
| 1332 | val_checks_per_epoch = self.num_training_batches // self.val_check_batch |
| 1333 | val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0 |
| 1334 | total_val_batches = self.num_val_batches * val_checks_per_epoch |
| 1335 | |
| 1336 | # total batches includes multiple val checks |
| 1337 | self.total_batches = self.num_training_batches + total_val_batches |
| 1338 | self.batch_loss_value = 0 # accumulated grads |
| 1339 | |
| 1340 | if self.is_iterable_train_dataloader: |
| 1341 | # for iterable train loader, the progress bar never ends |
| 1342 | num_iterations = None |
| 1343 | else: |
| 1344 | num_iterations = self.total_batches |
| 1345 | |
| 1346 | # reset progress bar |
| 1347 | # .reset() doesn't work on disabled progress bar so we should check |
| 1348 | desc = f'Epoch {epoch + 1}' if not self.is_iterable_train_dataloader else '' |
| 1349 | self.main_progress_bar.set_description(desc) |
| 1350 | |
| 1351 | # changing gradient according accumulation_scheduler |
| 1352 | self.accumulation_scheduler.on_epoch_begin(epoch, self) |
| 1353 | |
| 1354 | # ----------------- |
| 1355 | # RUN TNG EPOCH |
| 1356 | # ----------------- |
| 1357 | self.run_training_epoch() |
| 1358 | |
| 1359 | # update LR schedulers |
| 1360 | if self.lr_schedulers is not None: |
| 1361 | for lr_scheduler in self.lr_schedulers: |
| 1362 | lr_scheduler.step(epoch=self.current_epoch) |
| 1363 | |
| 1364 | self.main_progress_bar.close() |
| 1365 | |
| 1366 | model.on_train_end() |
| 1367 | |
| 1368 | if self.logger is not None: |
| 1369 | self.logger.finalize("success") |
| 1370 |
no test coverage detected