A class which is responsible for stepping the environment with epsilon-greedy, and fill the results to experience replay buffer.
| 110 | |
| 111 | |
| 112 | class EnvRunner(object): |
| 113 | """ |
| 114 | A class which is responsible for |
| 115 | stepping the environment with epsilon-greedy, |
| 116 | and fill the results to experience replay buffer. |
| 117 | """ |
| 118 | def __init__(self, player, predictor, memory, history_len): |
| 119 | """ |
| 120 | Args: |
| 121 | player (gym.Env) |
| 122 | predictor (callable): the model forward function which takes a |
| 123 | state and returns the prediction. |
| 124 | memory (ReplayMemory): the replay memory to store experience to. |
| 125 | history_len (int): |
| 126 | """ |
| 127 | self.player = player |
| 128 | self.num_actions = player.action_space.n |
| 129 | self.predictor = predictor |
| 130 | self.memory = memory |
| 131 | self.state_shape = memory.state_shape |
| 132 | self.dtype = memory.dtype |
| 133 | self.history_len = history_len |
| 134 | |
| 135 | self._current_episode = [] |
| 136 | self._current_ob = player.reset() |
| 137 | self._current_game_score = StatCounter() # store per-step reward |
| 138 | self.total_scores = [] # store per-game total score |
| 139 | |
| 140 | self.rng = get_rng(self) |
| 141 | |
| 142 | def step(self, exploration): |
| 143 | """ |
| 144 | Run the environment for one step. |
| 145 | If the episode ends, store the entire episode to the replay memory. |
| 146 | """ |
| 147 | old_s = self._current_ob |
| 148 | if self.rng.rand() <= exploration: |
| 149 | act = self.rng.choice(range(self.num_actions)) |
| 150 | else: |
| 151 | history = self.recent_state() |
| 152 | history.append(old_s) |
| 153 | history = np.stack(history, axis=-1) # state_shape + (Hist,) |
| 154 | |
| 155 | # assume batched network |
| 156 | history = np.expand_dims(history, axis=0) |
| 157 | q_values = self.predictor(history)[0][0] # this is the bottleneck |
| 158 | act = np.argmax(q_values) |
| 159 | |
| 160 | self._current_ob, reward, isOver, info = self.player.step(act) |
| 161 | self._current_game_score.feed(reward) |
| 162 | self._current_episode.append(Experience(old_s, act, reward, isOver)) |
| 163 | |
| 164 | if isOver: |
| 165 | flush_experience = True |
| 166 | if 'ale.lives' in info: # if running Atari, do something special |
| 167 | if info['ale.lives'] != 0: |
| 168 | # only record score and flush experience |
| 169 | # when a whole game is over (not when an episode is over) |
no outgoing calls
no test coverage detected
searching dependent graphs…