Estimate the time left until completion of training.
| 82 | |
| 83 | |
| 84 | class EstimatedTimeLeft(Callback): |
| 85 | """ |
| 86 | Estimate the time left until completion of training. |
| 87 | """ |
| 88 | def __init__(self, last_k_epochs=5, median=True): |
| 89 | """ |
| 90 | Args: |
| 91 | last_k_epochs (int): Use the time spent on last k epochs to estimate total time left. |
| 92 | median (bool): Use the mean or median time spent on last k epochs. |
| 93 | """ |
| 94 | self._times = deque(maxlen=last_k_epochs) |
| 95 | self._median = median |
| 96 | |
| 97 | def _before_train(self): |
| 98 | self._max_epoch = self.trainer.max_epoch |
| 99 | self._last_time = time.time() |
| 100 | |
| 101 | def _trigger_epoch(self): |
| 102 | duration = time.time() - self._last_time |
| 103 | self._last_time = time.time() |
| 104 | self._times.append(duration) |
| 105 | |
| 106 | epoch_time = np.median(self._times) if self._median else np.mean(self._times) |
| 107 | time_left = (self._max_epoch - self.epoch_num) * epoch_time |
| 108 | if time_left > 0: |
| 109 | logger.info("Estimated Time Left: " + humanize_time_delta(time_left)) |
no outgoing calls
no test coverage detected