| 35 | |
| 36 | # Holds one SGDRegressor for each action |
| 37 | class Model: |
| 38 | def __init__(self, env, feature_transformer): |
| 39 | self.env = env |
| 40 | self.models = [] |
| 41 | self.feature_transformer = feature_transformer |
| 42 | |
| 43 | sample_feature = feature_transformer.transform( [env.reset()] ) |
| 44 | D = sample_feature.shape[1] |
| 45 | |
| 46 | for i in range(env.action_space.n): |
| 47 | # model = SGDRegressor(learning_rate="constant") |
| 48 | # model.partial_fit(feature_transformer.transform( [env.reset()] ), [0]) |
| 49 | model = SGDRegressor(D) |
| 50 | self.models.append(model) |
| 51 | |
| 52 | self.eligibilities = np.zeros((env.action_space.n, D)) |
| 53 | |
| 54 | def reset(self): |
| 55 | self.eligibilities = np.zeros_like(self.eligibilities) |
| 56 | |
| 57 | def predict(self, s): |
| 58 | X = self.feature_transformer.transform([s]) |
| 59 | result = np.stack([m.predict(X) for m in self.models]).T |
| 60 | return result |
| 61 | |
| 62 | def update(self, s, a, G, gamma, lambda_): |
| 63 | X = self.feature_transformer.transform([s]) |
| 64 | # assert(len(X.shape) == 2) |
| 65 | |
| 66 | # slower |
| 67 | # for action in range(self.env.action_space.n): |
| 68 | # if action != a: |
| 69 | # self.eligibilities[action] *= gamma*lambda_ |
| 70 | # else: |
| 71 | # self.eligibilities[a] = grad + gamma*lambda_*self.eligibilities[a] |
| 72 | |
| 73 | self.eligibilities *= gamma*lambda_ |
| 74 | self.eligibilities[a] += X[0] |
| 75 | self.models[a].partial_fit(X[0], G, self.eligibilities[a]) |
| 76 | |
| 77 | def sample_action(self, s, eps): |
| 78 | if np.random.random() < eps: |
| 79 | return self.env.action_space.sample() |
| 80 | else: |
| 81 | return np.argmax(self.predict(s)) |
| 82 | |
| 83 | |
| 84 | # returns a list of states_and_rewards, and the total reward |