Method __init__

rl2/mountaincar/pg_theano.py:67–129

(self, D, ft, hidden_layer_sizes=[])

# approximates pi(a | s)
class PolicyModel:
  def __init__(self, D, ft, hidden_layer_sizes=[]):
    self.ft = ft

    ##### hidden layers #####
    M1 = D
    self.hidden_layers = []
    for M2 in hidden_layer_sizes:
      layer = HiddenLayer(M1, M2)
      self.hidden_layers.append(layer)
      M1 = M2

    # final layer mean
    self.mean_layer = HiddenLayer(M1, 1, lambda x: x, use_bias=False, zeros=True)

    # final layer variance
    self.var_layer = HiddenLayer(M1, 1, T.nnet.softplus, use_bias=False, zeros=False)

    # get all params for gradient later
    params = self.mean_layer.params + self.var_layer.params
    for layer in self.hidden_layers:
      params += layer.params

    # inputs and targets
    X = T.matrix('X')
    actions = T.vector('actions')
    advantages = T.vector('advantages')
    target_value = T.vector('target_value')

    # get final hidden layer
    Z = X
    for layer in self.hidden_layers:
      Z = layer.forward(Z)

    mean = self.mean_layer.forward(Z).flatten()
    var = self.var_layer.forward(Z).flatten() + 1e-5 # smoothing

    # can't find Theano log pdf, we will make it
    def log_pdf(actions, mean, var):
      k1 = T.log(2*np.pi*var)
      k2 = (actions - mean)**2 / var
      return -0.5*(k1 + k2)

    def entropy(var):
      return 0.5*T.log(2*np.pi*np.e*var)

    log_probs = log_pdf(actions, mean, var)
    cost = -T.sum(advantages * log_probs + 0.1*entropy(var))
    updates = adam(cost, params)

    # compile functions
    self.train_op = theano.function(
      inputs=[X, actions, advantages],
      updates=updates,
      allow_input_downcast=True
    )

    # alternatively, we could create a RandomStream and sample from
    # the Gaussian using Theano code