Class TRPO

examples/reinforcement_learning/tutorial_TRPO.py:180–437

trpo class

class TRPO:
    """
    trpo class
    """

    def __init__(self, state_dim, action_dim, action_bound):
        # critic
        with tf.name_scope('critic'):
            layer = input_layer = tl.layers.Input([None, state_dim], tf.float32)
            for d in HIDDEN_SIZES:
                layer = tl.layers.Dense(d, tf.nn.relu)(layer)
            v = tl.layers.Dense(1)(layer)
        self.critic = tl.models.Model(input_layer, v)
        self.critic.train()

        # actor
        with tf.name_scope('actor'):
            layer = input_layer = tl.layers.Input([None, state_dim], tf.float32)
            for d in HIDDEN_SIZES:
                layer = tl.layers.Dense(d, tf.nn.relu)(layer)
            mean = tl.layers.Dense(action_dim, tf.nn.tanh)(layer)
            mean = tl.layers.Lambda(lambda x: x * action_bound)(mean)
            log_std = tf.Variable(np.zeros(action_dim, dtype=np.float32))

        self.actor = tl.models.Model(input_layer, mean)
        self.actor.trainable_weights.append(log_std)
        self.actor.log_std = log_std
        self.actor.train()

        self.buf = GAE_Buffer(state_dim, action_dim, BATCH_SIZE, GAMMA, LAM)
        self.critic_optimizer = tf.optimizers.Adam(learning_rate=VF_LR)
        self.action_bound = action_bound

    def get_action(self, state, greedy=False):
        """
        get action
        :param state: state input
        :param greedy: get action greedy or not
        :return: pi, v, logp_pi, mean, log_std
        """
        state = np.array([state], np.float32)
        mean = self.actor(state)
        log_std = tf.convert_to_tensor(self.actor.log_std)
        std = tf.exp(log_std)
        std = tf.ones_like(mean) * std
        pi = tfp.distributions.Normal(mean, std)

        if greedy:
            action = mean
        else:
            action = pi.sample()
        action = np.clip(action, -self.action_bound, self.action_bound)
        logp_pi = pi.log_prob(action)

        value = self.critic(state)
        return action[0], value, logp_pi, mean, log_std

    def pi_loss(self, states, actions, adv, old_log_prob):