| 47 | |
| 48 | # approximates pi(a | s) |
| 49 | class PolicyModel: |
| 50 | def __init__(self, ft, D, hidden_layer_sizes_mean=[], hidden_layer_sizes_var=[]): |
| 51 | # save inputs for copy |
| 52 | self.ft = ft |
| 53 | self.D = D |
| 54 | self.hidden_layer_sizes_mean = hidden_layer_sizes_mean |
| 55 | self.hidden_layer_sizes_var = hidden_layer_sizes_var |
| 56 | |
| 57 | ##### model the mean ##### |
| 58 | self.mean_layers = [] |
| 59 | M1 = D |
| 60 | for M2 in hidden_layer_sizes_mean: |
| 61 | layer = HiddenLayer(M1, M2) |
| 62 | self.mean_layers.append(layer) |
| 63 | M1 = M2 |
| 64 | |
| 65 | # final layer |
| 66 | layer = HiddenLayer(M1, 1, lambda x: x, use_bias=False, zeros=True) |
| 67 | self.mean_layers.append(layer) |
| 68 | |
| 69 | |
| 70 | ##### model the variance ##### |
| 71 | self.var_layers = [] |
| 72 | M1 = D |
| 73 | for M2 in hidden_layer_sizes_var: |
| 74 | layer = HiddenLayer(M1, M2) |
| 75 | self.var_layers.append(layer) |
| 76 | M1 = M2 |
| 77 | |
| 78 | # final layer |
| 79 | layer = HiddenLayer(M1, 1, T.nnet.softplus, use_bias=False, zeros=False) |
| 80 | self.var_layers.append(layer) |
| 81 | |
| 82 | # get all params for gradient later |
| 83 | params = [] |
| 84 | for layer in (self.mean_layers + self.var_layers): |
| 85 | params += layer.params |
| 86 | self.params = params |
| 87 | |
| 88 | # inputs and targets |
| 89 | X = T.matrix('X') |
| 90 | actions = T.vector('actions') |
| 91 | advantages = T.vector('advantages') |
| 92 | |
| 93 | # calculate output and cost |
| 94 | def get_output(layers): |
| 95 | Z = X |
| 96 | for layer in layers: |
| 97 | Z = layer.forward(Z) |
| 98 | return Z.flatten() |
| 99 | |
| 100 | mean = get_output(self.mean_layers) |
| 101 | var = get_output(self.var_layers) + 1e-4 # smoothing |
| 102 | |
| 103 | # alternatively, we could create a RandomStream and sample from |
| 104 | # the Gaussian using Theano code |
| 105 | self.predict_op = theano.function( |
| 106 | inputs=[X], |