| 43 | |
| 44 | # approximates pi(a | s) |
| 45 | class PolicyModel: |
| 46 | def __init__(self, D, K, hidden_layer_sizes): |
| 47 | # create the graph |
| 48 | # K = number of actions |
| 49 | self.layers = [] |
| 50 | M1 = D |
| 51 | for M2 in hidden_layer_sizes: |
| 52 | layer = HiddenLayer(M1, M2) |
| 53 | self.layers.append(layer) |
| 54 | M1 = M2 |
| 55 | |
| 56 | # final layer |
| 57 | # layer = HiddenLayer(M1, K, lambda x: x, use_bias=False) |
| 58 | layer = HiddenLayer(M1, K, tf.nn.softmax, use_bias=False) |
| 59 | self.layers.append(layer) |
| 60 | |
| 61 | # inputs and targets |
| 62 | self.X = tf.placeholder(tf.float32, shape=(None, D), name='X') |
| 63 | self.actions = tf.placeholder(tf.int32, shape=(None,), name='actions') |
| 64 | self.advantages = tf.placeholder(tf.float32, shape=(None,), name='advantages') |
| 65 | |
| 66 | # calculate output and cost |
| 67 | Z = self.X |
| 68 | for layer in self.layers: |
| 69 | Z = layer.forward(Z) |
| 70 | p_a_given_s = Z |
| 71 | # action_scores = Z |
| 72 | # p_a_given_s = tf.nn.softmax(action_scores) |
| 73 | # self.action_scores = action_scores |
| 74 | self.predict_op = p_a_given_s |
| 75 | |
| 76 | # self.one_hot_actions = tf.one_hot(self.actions, K) |
| 77 | |
| 78 | selected_probs = tf.log( |
| 79 | tf.reduce_sum( |
| 80 | p_a_given_s * tf.one_hot(self.actions, K), |
| 81 | reduction_indices=[1] |
| 82 | ) |
| 83 | ) |
| 84 | |
| 85 | # self.selected_probs = selected_probs |
| 86 | cost = -tf.reduce_sum(self.advantages * selected_probs) |
| 87 | # self.cost = cost |
| 88 | # self.train_op = tf.train.AdamOptimizer(1e-1).minimize(cost) |
| 89 | self.train_op = tf.train.AdagradOptimizer(1e-1).minimize(cost) |
| 90 | # self.train_op = tf.train.MomentumOptimizer(1e-4, momentum=0.9).minimize(cost) |
| 91 | # self.train_op = tf.train.GradientDescentOptimizer(1e-4).minimize(cost) |
| 92 | |
| 93 | def set_session(self, session): |
| 94 | self.session = session |
| 95 | |
| 96 | def partial_fit(self, X, actions, advantages): |
| 97 | X = np.atleast_2d(X) |
| 98 | actions = np.atleast_1d(actions) |
| 99 | advantages = np.atleast_1d(advantages) |
| 100 | self.session.run( |
| 101 | self.train_op, |
| 102 | feed_dict={ |