Ppl metric
| 24 | |
| 25 | |
| 26 | class PPLMetric(Metric): |
| 27 | """ |
| 28 | Ppl metric |
| 29 | """ |
| 30 | |
| 31 | def __init__(self, data_length): |
| 32 | super(PPLMetric, self).__init__() |
| 33 | self.clear() |
| 34 | self.data_length = data_length |
| 35 | pipeline_stages = context.get_auto_parallel_context("pipeline_stages") |
| 36 | per_stage_device_num = get_group_size() // pipeline_stages |
| 37 | stage_id = get_rank() // per_stage_device_num |
| 38 | self.is_last_stage = (stage_id == pipeline_stages - 1) |
| 39 | |
| 40 | def clear(self): |
| 41 | """Clear the internal evaluation result.""" |
| 42 | self.PPL = [] |
| 43 | self.tokens_count = 0 |
| 44 | |
| 45 | def update(self, *inputs): # inputs |
| 46 | """Update list of ppl""" |
| 47 | if not self.is_last_stage: |
| 48 | return |
| 49 | logits = inputs[0].asnumpy().flatten().tolist() # logits |
| 50 | self.PPL.append(logits[0] * self.data_length) |
| 51 | self.tokens_count += 1 |
| 52 | |
| 53 | def eval(self): |
| 54 | if not self.is_last_stage: |
| 55 | return 0 |
| 56 | if self.tokens_count == 0: |
| 57 | print("Warning: tokens_count is 0") |
| 58 | return 0 |
| 59 | val_loss = sum(self.PPL) / (self.tokens_count * self.data_length) |
| 60 | ppl = math.exp(min(20, val_loss)) |
| 61 | # print("====" * 20 + " ppl end") |
| 62 | # print("====" * 20 + " ppl: {}".format(ppl)) |
| 63 | # return ppl |
| 64 | return val_loss |
| 65 | |
| 66 | |
| 67 | class ValidationLoss(Metric): |
no outgoing calls
no test coverage detected