Monitor the loss in training. If the loss in NAN or INF terminating training.
| 29 | |
| 30 | |
| 31 | class LossCallBack(Callback): |
| 32 | """ |
| 33 | Monitor the loss in training. |
| 34 | If the loss in NAN or INF terminating training. |
| 35 | """ |
| 36 | |
| 37 | def __init__( |
| 38 | self, |
| 39 | name, |
| 40 | dataset_size=-1, |
| 41 | local_rank=0, |
| 42 | rank_size=1, |
| 43 | has_trained_epoch=0, |
| 44 | has_trained_step=0, |
| 45 | micro_size=1, |
| 46 | sink_size=2, |
| 47 | tb_writer=None, |
| 48 | ): |
| 49 | super(LossCallBack, self).__init__() |
| 50 | self._dataset_size = dataset_size |
| 51 | self.local_rank = local_rank |
| 52 | self.rank_size = rank_size |
| 53 | self.has_trained_epoch = has_trained_epoch |
| 54 | self.has_trained_step = has_trained_step |
| 55 | self.micro_size = micro_size |
| 56 | self.sink_size = sink_size |
| 57 | |
| 58 | self.summary_writer = tb_writer |
| 59 | print("load has trained epoch :{} and step: {}".format(has_trained_epoch, has_trained_step), flush=True) |
| 60 | |
| 61 | def step_end(self, run_context): |
| 62 | """ |
| 63 | Print loss after each step |
| 64 | """ |
| 65 | cb_params = run_context.original_args() |
| 66 | if self._dataset_size > 0 and self.local_rank % 8 == 0: |
| 67 | percent, epoch_num = math.modf(cb_params.cur_step_num / |
| 68 | self._dataset_size) |
| 69 | if percent == 0: |
| 70 | epoch_num -= 1 |
| 71 | date = time.asctime(time.localtime(time.time())) |
| 72 | loss_value = cb_params.net_outputs[0].asnumpy() / self.micro_size |
| 73 | |
| 74 | if self.summary_writer is not None: |
| 75 | print(f"writing: {loss_value.item()}, {cb_params.net_outputs[2].asnumpy()}") |
| 76 | self.summary_writer.add_scalar( |
| 77 | tag="training_loss", |
| 78 | scalar_value=loss_value.item(), |
| 79 | global_step=cb_params.cur_step_num |
| 80 | + int(self.has_trained_step), |
| 81 | ) |
| 82 | self.summary_writer.add_scalar( |
| 83 | tag="loss_scale", |
| 84 | scalar_value=cb_params.net_outputs[2].asnumpy(), |
| 85 | global_step=cb_params.cur_step_num |
| 86 | + int(self.has_trained_step), |
| 87 | ) |
| 88 | print( |
no outgoing calls
no test coverage detected