Run the environment for one step. If the episode ends, store the entire episode to the replay memory.
(self, exploration)
| 140 | self.rng = get_rng(self) |
| 141 | |
| 142 | def step(self, exploration): |
| 143 | """ |
| 144 | Run the environment for one step. |
| 145 | If the episode ends, store the entire episode to the replay memory. |
| 146 | """ |
| 147 | old_s = self._current_ob |
| 148 | if self.rng.rand() <= exploration: |
| 149 | act = self.rng.choice(range(self.num_actions)) |
| 150 | else: |
| 151 | history = self.recent_state() |
| 152 | history.append(old_s) |
| 153 | history = np.stack(history, axis=-1) # state_shape + (Hist,) |
| 154 | |
| 155 | # assume batched network |
| 156 | history = np.expand_dims(history, axis=0) |
| 157 | q_values = self.predictor(history)[0][0] # this is the bottleneck |
| 158 | act = np.argmax(q_values) |
| 159 | |
| 160 | self._current_ob, reward, isOver, info = self.player.step(act) |
| 161 | self._current_game_score.feed(reward) |
| 162 | self._current_episode.append(Experience(old_s, act, reward, isOver)) |
| 163 | |
| 164 | if isOver: |
| 165 | flush_experience = True |
| 166 | if 'ale.lives' in info: # if running Atari, do something special |
| 167 | if info['ale.lives'] != 0: |
| 168 | # only record score and flush experience |
| 169 | # when a whole game is over (not when an episode is over) |
| 170 | flush_experience = False |
| 171 | self.player.reset() |
| 172 | |
| 173 | if flush_experience: |
| 174 | self.total_scores.append(self._current_game_score.sum) |
| 175 | self._current_game_score.reset() |
| 176 | |
| 177 | # Ensure that the whole episode of experience is continuous in the replay buffer |
| 178 | with self.memory.writer_lock: |
| 179 | for exp in self._current_episode: |
| 180 | self.memory.append(exp) |
| 181 | self._current_episode.clear() |
| 182 | |
| 183 | def recent_state(self): |
| 184 | """ |
no test coverage detected