Method update

examples/reinforcement_learning/tutorial_TRPO.py:390–423

update trpo :return: None

(self)

        return hvp

    def update(self):
        """
        update trpo
        :return: None
        """
        states, actions, adv, rewards_to_go, logp_old_ph, old_mu, old_log_std = self.buf.get()
        g, pi_l_old = self.gradient(states, actions, adv, logp_old_ph)

        Hx = lambda x: self.hvp(states, old_mu, old_log_std, x)
        x = self.cg(Hx, g)

        alpha = np.sqrt(2 * DELTA / (np.dot(x, Hx(x)) + EPS))
        old_params = self.get_pi_params()

        def set_and_eval(step):
            params = old_params - alpha * x * step
            self.set_pi_params(params)
            d_kl = self.kl(states, old_mu, old_log_std)
            loss = self.pi_loss(states, actions, adv, logp_old_ph)
            return [d_kl, loss]

        # trpo with backtracking line search, hard kl
        for j in range(BACKTRACK_ITERS):
            kl, pi_l_new = set_and_eval(step=BACKTRACK_COEFF**j)
            if kl <= DELTA and pi_l_new <= pi_l_old:
                # Accepting new params at step of line search
                break
        else:
            # Line search failed! Keeping old params.
            set_and_eval(step=0.)

        # Value function updates
        for _ in range(TRAIN_VF_ITERS):
            self.train_vf(states, rewards_to_go)

    def finish_path(self, done, next_state):
        """