| 626 | |
| 627 | |
| 628 | class MetricLogger: |
| 629 | def __init__(self, save_dir): |
| 630 | self.save_log = save_dir / "log" |
| 631 | with open(self.save_log, "w") as f: |
| 632 | f.write( |
| 633 | f"{'Episode':>8}{'Step':>8}{'Epsilon':>10}{'MeanReward':>15}" |
| 634 | f"{'MeanLength':>15}{'MeanLoss':>15}{'MeanQValue':>15}" |
| 635 | f"{'TimeDelta':>15}{'Time':>20}\n" |
| 636 | ) |
| 637 | self.ep_rewards_plot = save_dir / "reward_plot.jpg" |
| 638 | self.ep_lengths_plot = save_dir / "length_plot.jpg" |
| 639 | self.ep_avg_losses_plot = save_dir / "loss_plot.jpg" |
| 640 | self.ep_avg_qs_plot = save_dir / "q_plot.jpg" |
| 641 | |
| 642 | # History metrics |
| 643 | self.ep_rewards = [] |
| 644 | self.ep_lengths = [] |
| 645 | self.ep_avg_losses = [] |
| 646 | self.ep_avg_qs = [] |
| 647 | |
| 648 | # Moving averages, added for every call to record() |
| 649 | self.moving_avg_ep_rewards = [] |
| 650 | self.moving_avg_ep_lengths = [] |
| 651 | self.moving_avg_ep_avg_losses = [] |
| 652 | self.moving_avg_ep_avg_qs = [] |
| 653 | |
| 654 | # Current episode metric |
| 655 | self.init_episode() |
| 656 | |
| 657 | # Timing |
| 658 | self.record_time = time.time() |
| 659 | |
| 660 | def log_step(self, reward, loss, q): |
| 661 | self.curr_ep_reward += reward |
| 662 | self.curr_ep_length += 1 |
| 663 | if loss: |
| 664 | self.curr_ep_loss += loss |
| 665 | self.curr_ep_q += q |
| 666 | self.curr_ep_loss_length += 1 |
| 667 | |
| 668 | def log_episode(self): |
| 669 | "Mark end of episode" |
| 670 | self.ep_rewards.append(self.curr_ep_reward) |
| 671 | self.ep_lengths.append(self.curr_ep_length) |
| 672 | if self.curr_ep_loss_length == 0: |
| 673 | ep_avg_loss = 0 |
| 674 | ep_avg_q = 0 |
| 675 | else: |
| 676 | ep_avg_loss = np.round(self.curr_ep_loss / self.curr_ep_loss_length, 5) |
| 677 | ep_avg_q = np.round(self.curr_ep_q / self.curr_ep_loss_length, 5) |
| 678 | self.ep_avg_losses.append(ep_avg_loss) |
| 679 | self.ep_avg_qs.append(ep_avg_q) |
| 680 | |
| 681 | self.init_episode() |
| 682 | |
| 683 | def init_episode(self): |
| 684 | self.curr_ep_reward = 0.0 |
| 685 | self.curr_ep_length = 0 |