(self, net, param_init_net, param_info)
| 1626 | self.init_kwargs = kwargs |
| 1627 | |
| 1628 | def _run(self, net, param_init_net, param_info): |
| 1629 | param = param_info.blob |
| 1630 | grad = param_info.grad |
| 1631 | |
| 1632 | if self.alpha <= 0: |
| 1633 | return |
| 1634 | |
| 1635 | lr, iteration = self.build_lr( |
| 1636 | net, |
| 1637 | param_init_net, |
| 1638 | base_learning_rate=self.alpha, |
| 1639 | policy=self.policy, |
| 1640 | **(self.init_kwargs) |
| 1641 | ) |
| 1642 | |
| 1643 | m1 = param_init_net.ConstantFill([param], param + "_first_moment", value=0.0) |
| 1644 | |
| 1645 | if self.rowWise: |
| 1646 | shapes, types = workspace.InferShapesAndTypes([param_init_net]) |
| 1647 | m2 = param_init_net.ConstantFill( |
| 1648 | [], param + "_avg_second_moment", shape=[shapes[param][0]], value=0.0 |
| 1649 | ) |
| 1650 | else: |
| 1651 | m2 = param_init_net.ConstantFill( |
| 1652 | [param], param + "_second_moment", value=0.0 |
| 1653 | ) |
| 1654 | |
| 1655 | # Initialize "minibatch in which this parameter was last seen" for smart decay. |
| 1656 | if self.use_smart_decay: |
| 1657 | shapes, _ = workspace.InferShapesAndTypes([param_init_net]) |
| 1658 | last_seen = param_init_net.ConstantFill( |
| 1659 | [], param + "_last_seen", shape=[shapes[param][0]], value=0, dtype=core.DataType.INT64 |
| 1660 | ) |
| 1661 | self._aux_params.local.append(last_seen) |
| 1662 | |
| 1663 | self._aux_params.shared.append(iteration) |
| 1664 | self._aux_params.local.append(m1) |
| 1665 | self._aux_params.local.append(m2) |
| 1666 | |
| 1667 | if self.rowWise: |
| 1668 | assert isinstance(grad, core.GradientSlice), ( |
| 1669 | "If SparseAdam with rowWise=True, gradient must be " |
| 1670 | "a gradientslice. PLease ensure that rowWise is not enabled " |
| 1671 | "for the dense Adam optimizer, as it is not supported." |
| 1672 | ) |
| 1673 | |
| 1674 | output_blobs = [param, m1, m2] |
| 1675 | |
| 1676 | if self.use_smart_decay: |
| 1677 | output_blobs.append(last_seen) |
| 1678 | |
| 1679 | if self.use_lr_adaption: |
| 1680 | effective_grad = str(param) + "_effective_grad" |
| 1681 | output_blobs.append(effective_grad) |
| 1682 | |
| 1683 | if isinstance(grad, core.GradientSlice): |
| 1684 | grad = self.dedup(net, self.sparse_dedup_aggregator, grad) |
| 1685 | if self.rowWise: |
nothing calls this directly
no test coverage detected