| 121 | |
| 122 | |
| 123 | class LossSecondMomentResampler(LossAwareSampler): |
| 124 | def __init__(self, diffusion, history_per_term=10, uniform_prob=0.001): |
| 125 | self.diffusion = diffusion |
| 126 | self.history_per_term = history_per_term |
| 127 | self.uniform_prob = uniform_prob |
| 128 | self._loss_history = np.zeros( |
| 129 | [diffusion.num_timesteps, history_per_term], dtype=np.float64 |
| 130 | ) |
| 131 | self._loss_counts = np.zeros([diffusion.num_timesteps], dtype=np.int) |
| 132 | |
| 133 | def weights(self): |
| 134 | if not self._warmed_up(): |
| 135 | return np.ones([self.diffusion.num_timesteps], dtype=np.float64) |
| 136 | weights = np.sqrt(np.mean(self._loss_history ** 2, axis=-1)) |
| 137 | weights /= np.sum(weights) |
| 138 | weights *= 1 - self.uniform_prob |
| 139 | weights += self.uniform_prob / len(weights) |
| 140 | return weights |
| 141 | |
| 142 | def update_with_all_losses(self, ts, losses): |
| 143 | for t, loss in zip(ts, losses): |
| 144 | if self._loss_counts[t] == self.history_per_term: |
| 145 | # Shift out the oldest loss term. |
| 146 | self._loss_history[t, :-1] = self._loss_history[t, 1:] |
| 147 | self._loss_history[t, -1] = loss |
| 148 | else: |
| 149 | self._loss_history[t, self._loss_counts[t]] = loss |
| 150 | self._loss_counts[t] += 1 |
| 151 | |
| 152 | def _warmed_up(self): |
| 153 | return (self._loss_counts == self.history_per_term).all() |
| 154 | |
| 155 | |
| 156 | def mean_flat(tensor): |
no outgoing calls
no test coverage detected