| 65 | |
| 66 | |
| 67 | class ValidationLoss(Metric): |
| 68 | def __init__(self, data_length): |
| 69 | super(ValidationLoss, self).__init__() |
| 70 | self.clear() |
| 71 | self.data_length = data_length |
| 72 | pipeline_stages = context.get_auto_parallel_context("pipeline_stages") |
| 73 | per_stage_device_num = get_group_size() // pipeline_stages |
| 74 | stage_id = get_rank() // per_stage_device_num |
| 75 | self.is_last_stage = (stage_id == pipeline_stages - 1) |
| 76 | |
| 77 | def clear(self): |
| 78 | """Clear the internal evaluation result.""" |
| 79 | self.metric = [] |
| 80 | self.tokens_count = 0 |
| 81 | |
| 82 | def update(self, *inputs): # inputs |
| 83 | """Update list of ppl""" |
| 84 | # logits = inputs[0].asnumpy() |
| 85 | # if self.rank % 8 == 0: |
| 86 | # print("====" * 2 + " logits: {}".format(logits), flush=True) |
| 87 | # self.metric.append(logits) |
| 88 | if not self.is_last_stage: |
| 89 | return |
| 90 | logits = inputs[0].asnumpy().flatten().tolist() # logits |
| 91 | self.metric.append(logits[0] * self.data_length) |
| 92 | self.tokens_count += 1 |
| 93 | |
| 94 | def eval(self): |
| 95 | if not self.is_last_stage == 0: |
| 96 | return 0 |
| 97 | val_loss = sum(self.metric) / (self.tokens_count * self.data_length) |
| 98 | return val_loss |