| 50 | |
| 51 | # approximates pi(a | s) |
| 52 | class PolicyModel: |
| 53 | def __init__(self, ft, D, hidden_layer_sizes_mean=[], hidden_layer_sizes_var=[]): |
| 54 | |
| 55 | # save inputs for copy |
| 56 | self.ft = ft |
| 57 | self.D = D |
| 58 | self.hidden_layer_sizes_mean = hidden_layer_sizes_mean |
| 59 | self.hidden_layer_sizes_var = hidden_layer_sizes_var |
| 60 | |
| 61 | ##### model the mean ##### |
| 62 | self.mean_layers = [] |
| 63 | M1 = D |
| 64 | for M2 in hidden_layer_sizes_mean: |
| 65 | layer = HiddenLayer(M1, M2) |
| 66 | self.mean_layers.append(layer) |
| 67 | M1 = M2 |
| 68 | |
| 69 | # final layer |
| 70 | layer = HiddenLayer(M1, 1, lambda x: x, use_bias=False, zeros=True) |
| 71 | self.mean_layers.append(layer) |
| 72 | |
| 73 | |
| 74 | ##### model the variance ##### |
| 75 | self.var_layers = [] |
| 76 | M1 = D |
| 77 | for M2 in hidden_layer_sizes_var: |
| 78 | layer = HiddenLayer(M1, M2) |
| 79 | self.var_layers.append(layer) |
| 80 | M1 = M2 |
| 81 | |
| 82 | # final layer |
| 83 | layer = HiddenLayer(M1, 1, tf.nn.softplus, use_bias=False, zeros=False) |
| 84 | self.var_layers.append(layer) |
| 85 | |
| 86 | # gather params |
| 87 | self.params = [] |
| 88 | for layer in (self.mean_layers + self.var_layers): |
| 89 | self.params += layer.params |
| 90 | |
| 91 | # inputs and targets |
| 92 | self.X = tf.placeholder(tf.float32, shape=(None, D), name='X') |
| 93 | self.actions = tf.placeholder(tf.float32, shape=(None,), name='actions') |
| 94 | self.advantages = tf.placeholder(tf.float32, shape=(None,), name='advantages') |
| 95 | |
| 96 | def get_output(layers): |
| 97 | Z = self.X |
| 98 | for layer in layers: |
| 99 | Z = layer.forward(Z) |
| 100 | return tf.reshape(Z, [-1]) |
| 101 | |
| 102 | # calculate output and cost |
| 103 | mean = get_output(self.mean_layers) |
| 104 | std = get_output(self.var_layers) + 1e-4 # smoothing |
| 105 | |
| 106 | # note: the 'variance' is actually standard deviation |
| 107 | norm = tf.contrib.distributions.Normal(mean, std) |
| 108 | self.predict_op = tf.clip_by_value(norm.sample(), -1, 1) |
| 109 | |