5x5 grid with horizontally-bouncing obstacles (Deep SARSA, REINFORCE). Goal at (4,4) terminates; obstacle hit costs -1 but continues. State: 15-dim relative encoding (4 per obstacle, 3 for goal). Actions: 0=up, 1=down, 2=right, 3=left (note: differs from Env).
| 145 | |
| 146 | # --------------------------------------------------------------------------- |
| 147 | class DynamicEnv: |
| 148 | """5x5 grid with horizontally-bouncing obstacles (Deep SARSA, REINFORCE). |
| 149 | |
| 150 | Goal at (4,4) terminates; obstacle hit costs -1 but continues. |
| 151 | State: 15-dim relative encoding (4 per obstacle, 3 for goal). |
| 152 | Actions: 0=up, 1=down, 2=right, 3=left (note: differs from Env). |
| 153 | """ |
| 154 | n_actions = 4 |
| 155 | state_size = 15 |
| 156 | HUD = 32 |
| 157 | |
| 158 | def __init__(self, title="DynamicGridWorld", step_penalty=0.0, render_mode="human"): |
| 159 | self.title, self.step_penalty, self.render_mode = title, step_penalty, render_mode |
| 160 | self.agent = [0, 0] |
| 161 | self.obstacles_init = [[0, 1], [1, 2], [2, 3]] |
| 162 | self.goal = [4, 4] |
| 163 | self.obstacles = [] |
| 164 | self.counter, self.episode, self.score, self._hit = 0, 0, 0.0, 0 |
| 165 | self.last_score = None |
| 166 | self._screen = None |
| 167 | |
| 168 | def reset(self): |
| 169 | # Capture the prior episode's final score before wiping. |
| 170 | if self.counter > 0: |
| 171 | self.last_score = self.score |
| 172 | self.episode += 1 |
| 173 | self.agent, self.counter, self.score, self._hit = [0, 0], 0, 0.0, 0 |
| 174 | self.obstacles = [{"state": list(p), "direction": -1} for p in self.obstacles_init] |
| 175 | if self._screen is not None: |
| 176 | self.render(); time.sleep(0.3) |
| 177 | return self._state() |
| 178 | |
| 179 | def step(self, action): |
| 180 | self.counter += 1 |
| 181 | self.render() |
| 182 | if self.counter % 2 == 1: |
| 183 | for o in self.obstacles: |
| 184 | if o["state"][0] == WIDTH - 1: o["direction"] = 1 |
| 185 | elif o["state"][0] == 0: o["direction"] = -1 |
| 186 | o["state"][0] += 1 if o["direction"] == -1 else -1 |
| 187 | |
| 188 | x, y = self.agent |
| 189 | if action == 0 and y > 0: y -= 1 |
| 190 | elif action == 1 and y < HEIGHT - 1: y += 1 |
| 191 | elif action == 2 and x < WIDTH - 1: x += 1 |
| 192 | elif action == 3 and x > 0: x -= 1 |
| 193 | self.agent = [x, y] |
| 194 | |
| 195 | done = self.agent == self.goal |
| 196 | reward = 1.0 if done else sum(-1.0 for o in self.obstacles if o["state"] == self.agent) |
| 197 | reward -= self.step_penalty |
| 198 | self.score += reward |
| 199 | if reward < -self.step_penalty: |
| 200 | self._hit = 4 |
| 201 | return self._state(), reward, done |
| 202 | |
| 203 | def _state(self): |
| 204 | ax, ay = self.agent |
no outgoing calls
no test coverage detected