Computes and stores the average and current value
| 50 | |
| 51 | |
| 52 | class AverageMeter(object): |
| 53 | """Computes and stores the average and current value""" |
| 54 | |
| 55 | def __init__(self, name, fmt=":f", summary_type=Summary.AVERAGE): |
| 56 | self.name = name |
| 57 | self.fmt = fmt |
| 58 | self.summary_type = summary_type |
| 59 | self.reset() |
| 60 | |
| 61 | def reset(self): |
| 62 | self.val = 0 |
| 63 | self.avg = 0 |
| 64 | self.sum = 0 |
| 65 | self.count = 0 |
| 66 | |
| 67 | def update(self, val, n=1): |
| 68 | self.val = val |
| 69 | self.sum += val * n |
| 70 | self.count += n |
| 71 | self.avg = self.sum / self.count |
| 72 | |
| 73 | def all_reduce(self): |
| 74 | device = "cuda" if torch.cuda.is_available() else "cpu" |
| 75 | if isinstance(self.sum, np.ndarray): |
| 76 | total = torch.tensor( |
| 77 | self.sum.tolist() |
| 78 | + [ |
| 79 | self.count, |
| 80 | ], |
| 81 | dtype=torch.float32, |
| 82 | device=device, |
| 83 | ) |
| 84 | else: |
| 85 | total = torch.tensor( |
| 86 | [self.sum, self.count], dtype=torch.float32, device=device |
| 87 | ) |
| 88 | |
| 89 | dist.all_reduce(total, dist.ReduceOp.SUM, async_op=False) |
| 90 | if total.shape[0] > 2: |
| 91 | self.sum, self.count = total[:-1].cpu().numpy(), total[-1].cpu().item() |
| 92 | else: |
| 93 | self.sum, self.count = total.tolist() |
| 94 | self.avg = self.sum / (self.count + 1e-5) |
| 95 | |
| 96 | def __str__(self): |
| 97 | fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" |
| 98 | return fmtstr.format(**self.__dict__) |
| 99 | |
| 100 | def summary(self): |
| 101 | fmtstr = "" |
| 102 | if self.summary_type is Summary.NONE: |
| 103 | fmtstr = "" |
| 104 | elif self.summary_type is Summary.AVERAGE: |
| 105 | fmtstr = "{name} {avg:.3f}" |
| 106 | elif self.summary_type is Summary.SUM: |
| 107 | fmtstr = "{name} {sum:.3f}" |
| 108 | elif self.summary_type is Summary.COUNT: |
| 109 | fmtstr = "{name} {count:.3f}" |