| 37 | |
| 38 | class _SynchronizedBatchNorm(_BatchNorm): |
| 39 | def __init__(self, num_features, eps=1e-5, momentum=0.001, affine=True): |
| 40 | super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine) |
| 41 | |
| 42 | self._sync_master = SyncMaster(self._data_parallel_master) |
| 43 | |
| 44 | self._is_parallel = False |
| 45 | self._parallel_id = None |
| 46 | self._slave_pipe = None |
| 47 | |
| 48 | # customed batch norm statistics |
| 49 | self._moving_average_fraction = 1. - momentum |
| 50 | self.register_buffer('_tmp_running_mean', torch.zeros(self.num_features)) |
| 51 | self.register_buffer('_tmp_running_var', torch.ones(self.num_features)) |
| 52 | self.register_buffer('_running_iter', torch.ones(1)) |
| 53 | self._tmp_running_mean = self.running_mean.clone() * self._running_iter |
| 54 | self._tmp_running_var = self.running_var.clone() * self._running_iter |
| 55 | |
| 56 | def forward(self, input): |
| 57 | # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. |