| 54 | |
| 55 | # returns a list of states_and_rewards, and the total reward |
| 56 | def play_one(model, eps, gamma, n=5): |
| 57 | observation = env.reset()[0] |
| 58 | done = False |
| 59 | totalreward = 0 |
| 60 | rewards = [] |
| 61 | states = [] |
| 62 | actions = [] |
| 63 | iters = 0 |
| 64 | # array of [gamma^0, gamma^1, ..., gamma^(n-1)] |
| 65 | multiplier = np.array([gamma]*n)**np.arange(n) |
| 66 | # while not done and iters < 200: |
| 67 | while not done and iters < 10000: |
| 68 | # in earlier versions of gym, episode doesn't automatically |
| 69 | # end when you hit 200 steps |
| 70 | action = model.sample_action(observation, eps) |
| 71 | |
| 72 | states.append(observation) |
| 73 | actions.append(action) |
| 74 | |
| 75 | prev_observation = observation |
| 76 | observation, reward, done, truncated, info = env.step(action) |
| 77 | |
| 78 | rewards.append(reward) |
| 79 | |
| 80 | # update the model |
| 81 | if len(rewards) >= n: |
| 82 | # return_up_to_prediction = calculate_return_before_prediction(rewards, gamma) |
| 83 | return_up_to_prediction = multiplier.dot(rewards[-n:]) |
| 84 | action_values = model.predict(observation)[0] |
| 85 | # print("action_values.shape:", action_values.shape) |
| 86 | G = return_up_to_prediction + (gamma**n)*np.max(action_values) |
| 87 | # print("G:", G) |
| 88 | model.update(states[-n], actions[-n], G) |
| 89 | |
| 90 | # if len(rewards) > n: |
| 91 | # rewards.pop(0) |
| 92 | # states.pop(0) |
| 93 | # actions.pop(0) |
| 94 | # assert(len(rewards) <= n) |
| 95 | |
| 96 | totalreward += reward |
| 97 | iters += 1 |
| 98 | |
| 99 | # empty the cache |
| 100 | if n == 1: |
| 101 | rewards = [] |
| 102 | states = [] |
| 103 | actions = [] |
| 104 | else: |
| 105 | rewards = rewards[-n+1:] |
| 106 | states = states[-n+1:] |
| 107 | actions = actions[-n+1:] |
| 108 | # unfortunately, new version of gym cuts you off at 200 steps |
| 109 | # even if you haven't reached the goal. |
| 110 | # it's not good to do this UNLESS you've reached the goal. |
| 111 | # we are "really done" if position >= 0.5 |
| 112 | if observation[0] >= 0.5: |
| 113 | # we actually made it to the goal |