| 42 | |
| 43 | # Holds one BaseModel for each action |
| 44 | class Model: |
| 45 | def __init__(self, env, feature_transformer): |
| 46 | self.env = env |
| 47 | self.models = [] |
| 48 | self.feature_transformer = feature_transformer |
| 49 | |
| 50 | D = feature_transformer.dimensions |
| 51 | self.eligibilities = np.zeros((env.action_space.n, D)) |
| 52 | for i in range(env.action_space.n): |
| 53 | model = BaseModel(D) |
| 54 | self.models.append(model) |
| 55 | |
| 56 | def predict(self, s): |
| 57 | X = self.feature_transformer.transform([s]) |
| 58 | assert(len(X.shape) == 2) |
| 59 | result = np.stack([m.predict(X) for m in self.models]).T |
| 60 | assert(len(result.shape) == 2) |
| 61 | return result |
| 62 | |
| 63 | def update(self, s, a, G, gamma, lambda_): |
| 64 | X = self.feature_transformer.transform([s]) |
| 65 | assert(len(X.shape) == 2) |
| 66 | self.eligibilities *= gamma*lambda_ |
| 67 | self.eligibilities[a] += X[0] |
| 68 | self.models[a].partial_fit(X[0], G, self.eligibilities[a]) |
| 69 | |
| 70 | def sample_action(self, s, eps): |
| 71 | if np.random.random() < eps: |
| 72 | return self.env.action_space.sample() |
| 73 | else: |
| 74 | return np.argmax(self.predict(s)) |
| 75 | |
| 76 | |
| 77 | # returns a list of states_and_rewards, and the total reward |