§4 — Exponential Moving Average of model parameters. "We report sample quality metrics using an exponential moving average (EMA) of model parameters with a decay factor of 0.9999." The EMA weights are used for sampling/evaluation, not for training.
| 201 | |
| 202 | |
| 203 | class EMA: |
| 204 | """§4 — Exponential Moving Average of model parameters. |
| 205 | |
| 206 | "We report sample quality metrics using an exponential moving average (EMA) |
| 207 | of model parameters with a decay factor of 0.9999." |
| 208 | |
| 209 | The EMA weights are used for sampling/evaluation, not for training. |
| 210 | """ |
| 211 | |
| 212 | def __init__(self, model: nn.Module, decay: float = 0.9999): |
| 213 | """ |
| 214 | Args: |
| 215 | model: the model to track |
| 216 | decay: §4 — "decay factor of 0.9999" |
| 217 | """ |
| 218 | self.decay = decay |
| 219 | self.shadow = {} |
| 220 | for name, param in model.named_parameters(): |
| 221 | if param.requires_grad: |
| 222 | self.shadow[name] = param.data.clone() |
| 223 | |
| 224 | @torch.no_grad() |
| 225 | def update(self, model: nn.Module): |
| 226 | """Update EMA weights after each training step.""" |
| 227 | for name, param in model.named_parameters(): |
| 228 | if param.requires_grad and name in self.shadow: |
| 229 | self.shadow[name].mul_(self.decay).add_( |
| 230 | param.data, alpha=1.0 - self.decay |
| 231 | ) |
| 232 | |
| 233 | def apply(self, model: nn.Module): |
| 234 | """Load EMA weights into model (for evaluation/sampling).""" |
| 235 | for name, param in model.named_parameters(): |
| 236 | if name in self.shadow: |
| 237 | param.data.copy_(self.shadow[name]) |
| 238 | |
| 239 | def restore(self, model: nn.Module): |
| 240 | """Restore original model weights (after evaluation).""" |
| 241 | # NOTE: This requires storing original weights separately. |
| 242 | # The caller should save model.state_dict() before calling apply(). |
| 243 | pass |