(self, dev=cpu_dev)
| 150 | |
| 151 | @on_cpu_gpu |
| 152 | def test_Adam_const_lr(self, dev=cpu_dev): |
| 153 | cpu_dev.EnableGraph(False) |
| 154 | opt1 = opt.Adam(lr=0.1) |
| 155 | w_shape=(2,3) |
| 156 | w = tensor.Tensor(w_shape, device=dev).set_value(1.0) |
| 157 | g = tensor.Tensor(w_shape, device=dev).set_value(0.1) |
| 158 | |
| 159 | # m := beta_1 * m + (1 - beta_1) * grad |
| 160 | # v := beta_2 * v + (1 - beta_2) * grad * grad |
| 161 | # m_norm = m / (1 - beta_1 ^ step) |
| 162 | # v_norm = v / (1 - beta_2 ^ step) |
| 163 | # param := param - (lr * m_norm) / ( sqrt(v_norm) + epsilon) ) |
| 164 | |
| 165 | m = 0.1 * g |
| 166 | tmp = tensor.square(g) |
| 167 | v = 0.001 * tmp |
| 168 | |
| 169 | m_norm = m / 0.1 |
| 170 | v_norm = v / 0.001 |
| 171 | |
| 172 | tmp = tensor.sqrt(v_norm) + 1e-8 |
| 173 | tmp = m_norm / tmp |
| 174 | |
| 175 | w_step1 = w - 0.1 * tmp |
| 176 | opt1.apply(w.name, w, g) |
| 177 | |
| 178 | assertTensorEqual(w, w_step1, decimal=5) |
| 179 | |
| 180 | @on_cpu_gpu |
| 181 | def test_sgd_const_lr_momentum(self, dev=cpu_dev): |
nothing calls this directly
no test coverage detected