Method update

rl2/a3c/worker.py:219–269

Updates global policy and value networks using the local networks' gradients

(self, steps, sess)

    return steps, global_step

  def update(self, steps, sess):
    """
    Updates global policy and value networks using the local networks' gradients
    """

    # In order to accumulate the total return
    # We will use V_hat(s') to predict the future returns
    # But we will use the actual rewards if we have them
    # Ex. if we have s1, s2, s3 with rewards r1, r2, r3
    # Then G(s3) = r3 + V(s4)
    #      G(s2) = r2 + r3 + V(s4)
    #      G(s1) = r1 + r2 + r3 + V(s4)
    reward = 0.0
    if not steps[-1].done:
      reward = self.get_value_prediction(steps[-1].next_state, sess)

    # Accumulate minibatch samples
    states = []
    advantages = []
    value_targets = []
    actions = []

    # loop through steps in reverse order
    for step in reversed(steps):
      reward = step.reward + self.discount_factor * reward
      advantage = reward - self.get_value_prediction(step.state, sess)
      # Accumulate updates
      states.append(step.state)
      actions.append(step.action)
      advantages.append(advantage)
      value_targets.append(reward)

    feed_dict = {
      self.policy_net.states: np.array(states),
      self.policy_net.advantage: advantages,
      self.policy_net.actions: actions,
      self.value_net.states: np.array(states),
      self.value_net.targets: value_targets,
    }

    # Train the global estimators using local gradients
    global_step, pnet_loss, vnet_loss, _, _ = sess.run([
      self.global_step,
      self.policy_net.loss,
      self.value_net.loss,
      self.pnet_train_op,
      self.vnet_train_op,
    ], feed_dict)

    # Theoretically could plot these later
    return pnet_loss, vnet_loss