| 36 | |
| 37 | # Inspired by https://github.com/dennybritz/reinforcement-learning |
| 38 | class FeatureTransformer: |
| 39 | def __init__(self, env, n_components=500): |
| 40 | observation_examples = np.array([env.observation_space.sample() for x in range(10000)]) |
| 41 | scaler = StandardScaler() |
| 42 | scaler.fit(observation_examples) |
| 43 | |
| 44 | # Used to converte a state to a featurizes represenation. |
| 45 | # We use RBF kernels with different variances to cover different parts of the space |
| 46 | featurizer = FeatureUnion([ |
| 47 | ("rbf1", RBFSampler(gamma=5.0, n_components=n_components)), |
| 48 | ("rbf2", RBFSampler(gamma=2.0, n_components=n_components)), |
| 49 | ("rbf3", RBFSampler(gamma=1.0, n_components=n_components)), |
| 50 | ("rbf4", RBFSampler(gamma=0.5, n_components=n_components)) |
| 51 | ]) |
| 52 | example_features = featurizer.fit_transform(scaler.transform(observation_examples)) |
| 53 | |
| 54 | self.dimensions = example_features.shape[1] |
| 55 | self.scaler = scaler |
| 56 | self.featurizer = featurizer |
| 57 | |
| 58 | def transform(self, observations): |
| 59 | # print "observations:", observations |
| 60 | scaled = self.scaler.transform(observations) |
| 61 | # assert(len(scaled.shape) == 2) |
| 62 | return self.featurizer.transform(scaled) |
| 63 | |
| 64 | |
| 65 | # Holds one SGDRegressor for each action |