(model: torch.nn.Module, criterion: torch.nn.Module,
data_loader: Iterable, optimizer: torch.optim.Optimizer,
device: torch.device, epoch: int, max_norm: float = 0)
| 23 | |
| 24 | |
| 25 | def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, |
| 26 | data_loader: Iterable, optimizer: torch.optim.Optimizer, |
| 27 | device: torch.device, epoch: int, max_norm: float = 0): |
| 28 | model.train() |
| 29 | criterion.train() |
| 30 | metric_logger = utils.MetricLogger(delimiter=" ") |
| 31 | metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) |
| 32 | metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) |
| 33 | metric_logger.add_meter('grad_norm', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) |
| 34 | header = 'Epoch: [{}]'.format(epoch) |
| 35 | print_freq = 10 |
| 36 | |
| 37 | prefetcher = data_prefetcher(data_loader, device, prefetch=True) |
| 38 | samples, targets = prefetcher.next() |
| 39 | |
| 40 | # for samples, targets in metric_logger.log_every(data_loader, print_freq, header): |
| 41 | for _ in metric_logger.log_every(range(len(data_loader)), print_freq, header): |
| 42 | outputs = model(samples) |
| 43 | loss_dict = criterion(outputs, targets) |
| 44 | weight_dict = criterion.weight_dict |
| 45 | losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) |
| 46 | |
| 47 | # reduce losses over all GPUs for logging purposes |
| 48 | loss_dict_reduced = utils.reduce_dict(loss_dict) |
| 49 | loss_dict_reduced_unscaled = {f'{k}_unscaled': v |
| 50 | for k, v in loss_dict_reduced.items()} |
| 51 | loss_dict_reduced_scaled = {k: v * weight_dict[k] |
| 52 | for k, v in loss_dict_reduced.items() if k in weight_dict} |
| 53 | losses_reduced_scaled = sum(loss_dict_reduced_scaled.values()) |
| 54 | |
| 55 | loss_value = losses_reduced_scaled.item() |
| 56 | |
| 57 | if not math.isfinite(loss_value): |
| 58 | print("Loss is {}, stopping training".format(loss_value)) |
| 59 | print(loss_dict_reduced) |
| 60 | sys.exit(1) |
| 61 | |
| 62 | optimizer.zero_grad() |
| 63 | losses.backward() |
| 64 | if max_norm > 0: |
| 65 | grad_total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) |
| 66 | else: |
| 67 | grad_total_norm = utils.get_total_grad_norm(model.parameters(), max_norm) |
| 68 | optimizer.step() |
| 69 | |
| 70 | metric_logger.update(loss=loss_value, **loss_dict_reduced_scaled, **loss_dict_reduced_unscaled) |
| 71 | metric_logger.update(class_error=loss_dict_reduced['class_error']) |
| 72 | metric_logger.update(lr=optimizer.param_groups[0]["lr"]) |
| 73 | metric_logger.update(grad_norm=grad_total_norm) |
| 74 | |
| 75 | samples, targets = prefetcher.next() |
| 76 | # gather the stats from all processes |
| 77 | metric_logger.synchronize_between_processes() |
| 78 | print("Averaged stats:", metric_logger) |
| 79 | return {k: meter.global_avg for k, meter in metric_logger.meters.items()} |
| 80 | |
| 81 | |
| 82 | @torch.no_grad() |
no test coverage detected