| 166 | |
| 167 | |
| 168 | def learn(network, env, seed, new_session=True, nsteps=5, nstack=4, total_timesteps=int(80e6), |
| 169 | vf_coef=0.5, ent_coef=0.01, max_grad_norm=0.5, lr=7e-4, |
| 170 | epsilon=1e-5, alpha=0.99, gamma=0.99, log_interval=1000): |
| 171 | tf.reset_default_graph() |
| 172 | set_global_seeds(seed) |
| 173 | |
| 174 | nenvs = env.num_envs |
| 175 | env_id = env.env_id |
| 176 | save_name = os.path.join('models', env_id + '.save') |
| 177 | ob_space = env.observation_space |
| 178 | ac_space = env.action_space |
| 179 | agent = Agent(Network=network, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, |
| 180 | nsteps=nsteps, nstack=nstack, |
| 181 | ent_coef=ent_coef, vf_coef=vf_coef, |
| 182 | max_grad_norm=max_grad_norm, |
| 183 | lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps) |
| 184 | if os.path.exists(save_name): |
| 185 | agent.load(save_name) |
| 186 | |
| 187 | runner = Runner(env, agent, nsteps=nsteps, nstack=nstack, gamma=gamma) |
| 188 | |
| 189 | nbatch = nenvs * nsteps |
| 190 | tstart = time.time() |
| 191 | for update in range(1, total_timesteps // nbatch + 1): |
| 192 | states, rewards, actions, values = runner.run() |
| 193 | policy_loss, value_loss, policy_entropy = agent.train( |
| 194 | states, rewards, actions, values) |
| 195 | nseconds = time.time() - tstart |
| 196 | fps = int((update * nbatch) / nseconds) |
| 197 | if update % log_interval == 0 or update == 1: |
| 198 | print(' - - - - - - - ') |
| 199 | print("nupdates", update) |
| 200 | print("total_timesteps", update * nbatch) |
| 201 | print("fps", fps) |
| 202 | print("policy_entropy", float(policy_entropy)) |
| 203 | print("value_loss", float(value_loss)) |
| 204 | |
| 205 | # total reward |
| 206 | r = runner.total_rewards[-100:] # get last 100 |
| 207 | tr = runner.real_total_rewards[-100:] |
| 208 | if len(r) == 100: |
| 209 | print("avg reward (last 100):", np.mean(r)) |
| 210 | if len(tr) == 100: |
| 211 | print("avg total reward (last 100):", np.mean(tr)) |
| 212 | print("max (last 100):", np.max(tr)) |
| 213 | |
| 214 | agent.save(save_name) |
| 215 | |
| 216 | env.close() |
| 217 | agent.save(save_name) |