| 20 | |
| 21 | |
| 22 | class SAM(torch.optim.Optimizer): |
| 23 | def __init__(self, params, base_optimizer, rho=0.05, **kwargs): |
| 24 | assert rho >= 0.0, f"Invalid rho, should be non-negative: {rho}" |
| 25 | |
| 26 | defaults = dict(rho=rho, **kwargs) |
| 27 | super(SAM, self).__init__(params, defaults) |
| 28 | |
| 29 | self.base_optimizer = base_optimizer(self.param_groups, **kwargs) |
| 30 | self.param_groups = self.base_optimizer.param_groups |
| 31 | |
| 32 | @torch.no_grad() |
| 33 | def first_step(self, zero_grad=False): |
| 34 | grad_norm = self._grad_norm() |
| 35 | for group in self.param_groups: |
| 36 | scale = group["rho"] / (grad_norm + 1e-12) |
| 37 | |
| 38 | for p in group["params"]: |
| 39 | if p.grad is None: continue |
| 40 | e_w = p.grad * scale.to(p) |
| 41 | p.add_(e_w) # climb to the local maximum "w + e(w)" |
| 42 | self.state[p]["e_w"] = e_w |
| 43 | |
| 44 | if zero_grad: self.zero_grad() |
| 45 | |
| 46 | @torch.no_grad() |
| 47 | def second_step(self, zero_grad=False): |
| 48 | for group in self.param_groups: |
| 49 | for p in group["params"]: |
| 50 | if p.grad is None: continue |
| 51 | p.sub_(self.state[p]["e_w"]) # get back to "w" from "w + e(w)" |
| 52 | |
| 53 | self.base_optimizer.step() # do the actual "sharpness-aware" update |
| 54 | |
| 55 | if zero_grad: self.zero_grad() |
| 56 | |
| 57 | @torch.no_grad() |
| 58 | def step(self, closure=None): |
| 59 | assert closure is not None, "Sharpness Aware Minimization requires closure, but it was not provided" |
| 60 | closure = torch.enable_grad()(closure) # the closure should do a full forward-backward pass |
| 61 | |
| 62 | self.first_step(zero_grad=True) |
| 63 | closure() |
| 64 | self.second_step() |
| 65 | |
| 66 | def _grad_norm(self): |
| 67 | shared_device = self.param_groups[0]["params"][0].device # put everything on the same device, in case of model parallelism |
| 68 | norm = torch.norm( |
| 69 | torch.stack([ |
| 70 | p.grad.norm(p=2).to(shared_device) |
| 71 | for group in self.param_groups for p in group["params"] |
| 72 | if p.grad is not None |
| 73 | ]), |
| 74 | p=2 |
| 75 | ) |
| 76 | return norm |