Function momentum_updates

nlp_class2/glove_theano.py:27–38

(cost, params, lr=1e-4, mu=0.9)

def momentum_updates(cost, params, lr=1e-4, mu=0.9):
  grads = T.grad(cost, params)
  velocities = [theano.shared(
    np.zeros_like(p.get_value()).astype(np.float32)
  ) for p in params]
  updates = []
  for p, v, g in zip(params, velocities, grads):
    newv = mu*v - lr*g
    newp = p + newv
    updates.append((p, newp))
    updates.append((v, newv))
  return updates


class Glove: