| 90 | |
| 91 | |
| 92 | def evolution_strategy( |
| 93 | f, |
| 94 | population_size, |
| 95 | sigma, |
| 96 | lr, |
| 97 | initial_params, |
| 98 | num_iters): |
| 99 | |
| 100 | # assume initial params is a 1-D array |
| 101 | num_params = len(initial_params) |
| 102 | reward_per_iteration = np.zeros(num_iters) |
| 103 | |
| 104 | params = initial_params |
| 105 | for t in range(num_iters): |
| 106 | t0 = datetime.now() |
| 107 | N = np.random.randn(population_size, num_params) |
| 108 | |
| 109 | # ### slow way |
| 110 | # R = np.zeros(population_size) # stores the reward |
| 111 | |
| 112 | # # loop through each "offspring" |
| 113 | # for j in range(population_size): |
| 114 | # params_try = params + sigma*N[j] |
| 115 | # R[j] = f(params_try) |
| 116 | |
| 117 | ### fast way |
| 118 | R = pool.map(f, [params + sigma*N[j] for j in range(population_size)]) |
| 119 | R = np.array(R) |
| 120 | |
| 121 | m = R.mean() |
| 122 | s = R.std() |
| 123 | if s == 0: |
| 124 | # we can't apply the following equation |
| 125 | print("Skipping") |
| 126 | continue |
| 127 | |
| 128 | A = (R - m) / s |
| 129 | reward_per_iteration[t] = m |
| 130 | params = params + lr/(population_size*sigma) * np.dot(N.T, A) |
| 131 | |
| 132 | # update the learning rate |
| 133 | # lr *= 0.992354 |
| 134 | # sigma *= 0.99 |
| 135 | |
| 136 | print("Iter:", t, "Avg Reward: %.3f" % m, "Max:", R.max(), "Duration:", (datetime.now() - t0)) |
| 137 | |
| 138 | return params, reward_per_iteration |
| 139 | |
| 140 | |
| 141 | def reward_function(params, display=False): |