| 127 | |
| 128 | |
| 129 | def evolution_strategy( |
| 130 | f, |
| 131 | population_size, |
| 132 | sigma, |
| 133 | lr, |
| 134 | initial_params, |
| 135 | num_iters): |
| 136 | |
| 137 | # assume initial params is a 1-D array |
| 138 | num_params = len(initial_params) |
| 139 | reward_per_iteration = np.zeros(num_iters) |
| 140 | |
| 141 | params = initial_params |
| 142 | for t in range(num_iters): |
| 143 | t0 = datetime.now() |
| 144 | N = np.random.randn(population_size, num_params) |
| 145 | |
| 146 | ### slow way |
| 147 | R = np.zeros(population_size) # stores the reward |
| 148 | |
| 149 | # loop through each "offspring" |
| 150 | for j in range(population_size): |
| 151 | params_try = params + sigma*N[j] |
| 152 | R[j] = f(params_try) |
| 153 | |
| 154 | ### fast way |
| 155 | # R = pool.map(f, [params + sigma*N[j] for j in range(population_size)]) |
| 156 | # R = np.array(R) |
| 157 | |
| 158 | m = R.mean() |
| 159 | s = R.std() |
| 160 | if s == 0: |
| 161 | # we can't apply the following equation |
| 162 | print("Skipping") |
| 163 | continue |
| 164 | |
| 165 | A = (R - m) / s |
| 166 | reward_per_iteration[t] = m |
| 167 | params = params + lr/(population_size*sigma) * np.dot(N.T, A) |
| 168 | |
| 169 | # update the learning rate |
| 170 | lr *= 0.992354 |
| 171 | # sigma *= 0.99 |
| 172 | |
| 173 | print("Iter:", t, "Avg Reward: %.3f" % m, "Max:", R.max(), "Duration:", (datetime.now() - t0)) |
| 174 | |
| 175 | return params, reward_per_iteration |
| 176 | |
| 177 | |
| 178 | def reward_function(params): |