(self, state, action, futurereward, action_prob)
| 101 | return logits, value |
| 102 | |
| 103 | def build_graph(self, state, action, futurereward, action_prob): |
| 104 | logits, value = self._get_NN_prediction(state) |
| 105 | value = tf.squeeze(value, [1], name='pred_value') # (B,) |
| 106 | policy = tf.nn.softmax(logits, name='policy') |
| 107 | if not self.training: |
| 108 | return |
| 109 | log_probs = tf.log(policy + 1e-6) |
| 110 | |
| 111 | log_pi_a_given_s = tf.reduce_sum( |
| 112 | log_probs * tf.one_hot(action, NUM_ACTIONS), 1) |
| 113 | advantage = tf.subtract(tf.stop_gradient(value), futurereward, name='advantage') |
| 114 | |
| 115 | pi_a_given_s = tf.reduce_sum(policy * tf.one_hot(action, NUM_ACTIONS), 1) # (B,) |
| 116 | importance = tf.stop_gradient(tf.clip_by_value(pi_a_given_s / (action_prob + 1e-8), 0, 10)) |
| 117 | |
| 118 | policy_loss = tf.reduce_sum(log_pi_a_given_s * advantage * importance, name='policy_loss') |
| 119 | xentropy_loss = tf.reduce_sum(policy * log_probs, name='xentropy_loss') |
| 120 | value_loss = tf.nn.l2_loss(value - futurereward, name='value_loss') |
| 121 | |
| 122 | pred_reward = tf.reduce_mean(value, name='predict_reward') |
| 123 | advantage = tf.sqrt(tf.reduce_mean(tf.square(advantage)), name='rms_advantage') |
| 124 | entropy_beta = tf.get_variable('entropy_beta', shape=[], |
| 125 | initializer=tf.constant_initializer(0.01), trainable=False) |
| 126 | cost = tf.add_n([policy_loss, xentropy_loss * entropy_beta, value_loss]) |
| 127 | cost = tf.truediv(cost, tf.cast(tf.shape(futurereward)[0], tf.float32), name='cost') |
| 128 | summary.add_moving_summary(policy_loss, xentropy_loss, |
| 129 | value_loss, pred_reward, advantage, |
| 130 | cost, tf.reduce_mean(importance, name='importance')) |
| 131 | return cost |
| 132 | |
| 133 | def optimizer(self): |
| 134 | lr = tf.get_variable('learning_rate', initializer=0.001, trainable=False) |
nothing calls this directly
no test coverage detected