| 64 | |
| 65 | # Holds one SGDRegressor for each action |
| 66 | class Model: |
| 67 | def __init__(self, env, feature_transformer, learning_rate): |
| 68 | self.env = env |
| 69 | self.models = [] |
| 70 | self.feature_transformer = feature_transformer |
| 71 | for i in range(env.action_space.n): |
| 72 | model = SGDRegressor(learning_rate=learning_rate) |
| 73 | model.partial_fit(feature_transformer.transform( [env.reset()[0]] ), [0]) |
| 74 | self.models.append(model) |
| 75 | |
| 76 | def predict(self, s): |
| 77 | X = self.feature_transformer.transform([s]) |
| 78 | result = np.stack([m.predict(X) for m in self.models]).T |
| 79 | assert(len(result.shape) == 2) |
| 80 | return result |
| 81 | |
| 82 | def update(self, s, a, G): |
| 83 | X = self.feature_transformer.transform([s]) |
| 84 | assert(len(X.shape) == 2) |
| 85 | self.models[a].partial_fit(X, [G]) |
| 86 | |
| 87 | def sample_action(self, s, eps): |
| 88 | # eps = 0 |
| 89 | # Technically, we don't need to do epsilon-greedy |
| 90 | # because SGDRegressor predicts 0 for all states |
| 91 | # until they are updated. This works as the |
| 92 | # "Optimistic Initial Values" method, since all |
| 93 | # the rewards for Mountain Car are -1. |
| 94 | if np.random.random() < eps: |
| 95 | return self.env.action_space.sample() |
| 96 | else: |
| 97 | return np.argmax(self.predict(s)) |
| 98 | |
| 99 | |
| 100 | # returns a list of states_and_rewards, and the total reward |