(monkeypatch, case, update_lr, inplace_mode)
| 254 | @pytest.mark.parametrize("update_lr", [False, True]) |
| 255 | @pytest.mark.parametrize("inplace_mode", [False, True]) |
| 256 | def test_adadelta(monkeypatch, case, update_lr, inplace_mode): |
| 257 | class CheckValue: |
| 258 | def __init__(self, net, **kwarg): |
| 259 | self.s_slots = {} |
| 260 | self.a_slots = {} |
| 261 | for param in net.parameters(): |
| 262 | self.s_slots[param] = np.zeros(param.shape).astype(np.float32) |
| 263 | self.a_slots[param] = np.zeros(param.shape).astype(np.float32) |
| 264 | for k, v in kwarg.items(): |
| 265 | setattr(self, k, v) |
| 266 | |
| 267 | def __call__(self, ori_params, new_params, step): |
| 268 | for param in new_params: |
| 269 | grad = param.grad.numpy() |
| 270 | if hasattr(self, "weight_decay") and self.weight_decay != 0.0: |
| 271 | grad = grad + ori_params[param] * self.weight_decay |
| 272 | self.s_slots[param] = self.s_slots[param] * self.rho + grad ** 2 * ( |
| 273 | 1 - self.rho |
| 274 | ) |
| 275 | delta = ( |
| 276 | grad |
| 277 | * ((self.a_slots[param] + self.eps) ** 0.5) |
| 278 | / (self.s_slots[param] + self.eps) ** 0.5 |
| 279 | ) |
| 280 | self.a_slots[param] = self.a_slots[param] * self.rho + delta ** 2 * ( |
| 281 | 1 - self.rho |
| 282 | ) |
| 283 | delta *= -self.lr |
| 284 | np.testing.assert_almost_equal( |
| 285 | param.numpy(), ori_params[param] + delta, decimal=6 |
| 286 | ) |
| 287 | |
| 288 | with monkeypatch.context() as mk: |
| 289 | mk.setenv("MEGENGINE_INPLACE_UPDATE", str(int(inplace_mode))) |
| 290 | _test_optimizer("Adadelta", case, CheckValue, update_lr=update_lr) |
| 291 | |
| 292 | |
| 293 | @pytest.mark.parametrize( |
nothing calls this directly
no test coverage detected