* Train the policy network's model. * * @param {CartPole} cartPoleSystem The cart-pole system object to use during * training. * @param {tf.train.Optimizer} optimizer An instance of TensorFlow.js * Optimizer to use for training. * @param {number} discountRate Reward discounting
(
cartPoleSystem, optimizer, discountRate, numGames, maxStepsPerGame)
| 108 | * in this round of training. |
| 109 | */ |
| 110 | async train( |
| 111 | cartPoleSystem, optimizer, discountRate, numGames, maxStepsPerGame) { |
| 112 | const allGradients = []; |
| 113 | const allRewards = []; |
| 114 | const gameSteps = []; |
| 115 | onGameEnd(0, numGames); |
| 116 | for (let i = 0; i < numGames; ++i) { |
| 117 | // Randomly initialize the state of the cart-pole system at the beginning |
| 118 | // of every game. |
| 119 | cartPoleSystem.setRandomState(); |
| 120 | const gameRewards = []; |
| 121 | const gameGradients = []; |
| 122 | for (let j = 0; j < maxStepsPerGame; ++j) { |
| 123 | // For every step of the game, remember gradients of the policy |
| 124 | // network's weights with respect to the probability of the action |
| 125 | // choice that lead to the reward. |
| 126 | const gradients = tf.tidy(() => { |
| 127 | const inputTensor = cartPoleSystem.getStateTensor(); |
| 128 | return this.getGradientsAndSaveActions(inputTensor).grads; |
| 129 | }); |
| 130 | |
| 131 | this.pushGradients(gameGradients, gradients); |
| 132 | const action = this.currentActions_[0]; |
| 133 | const isDone = cartPoleSystem.update(action); |
| 134 | |
| 135 | await maybeRenderDuringTraining(cartPoleSystem); |
| 136 | |
| 137 | if (isDone) { |
| 138 | // When the game ends before max step count is reached, a reward of |
| 139 | // 0 is given. |
| 140 | gameRewards.push(0); |
| 141 | break; |
| 142 | } else { |
| 143 | // As long as the game doesn't end, each step leads to a reward of 1. |
| 144 | // These reward values will later be "discounted", leading to |
| 145 | // higher reward values for longer-lasting games. |
| 146 | gameRewards.push(1); |
| 147 | } |
| 148 | } |
| 149 | onGameEnd(i + 1, numGames); |
| 150 | gameSteps.push(gameRewards.length); |
| 151 | this.pushGradients(allGradients, gameGradients); |
| 152 | allRewards.push(gameRewards); |
| 153 | await tf.nextFrame(); |
| 154 | } |
| 155 | |
| 156 | tf.tidy(() => { |
| 157 | // The following line does three things: |
| 158 | // 1. Performs reward discounting, i.e., make recent rewards count more |
| 159 | // than rewards from the further past. The effect is that the reward |
| 160 | // values from a game with many steps become larger than the values |
| 161 | // from a game with fewer steps. |
| 162 | // 2. Normalize the rewards, i.e., subtract the global mean value of the |
| 163 | // rewards and divide the result by the global standard deviation of |
| 164 | // the rewards. Together with step 1, this makes the rewards from |
| 165 | // long-lasting games positive and rewards from short-lasting |
| 166 | // negative. |
| 167 | // 3. Scale the gradients with the normalized reward values. |