From fairseq
(lprobs, target, epsilon, ignore_index=-100)
| 45 | |
| 46 | |
| 47 | def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=-100): |
| 48 | """From fairseq""" |
| 49 | if target.dim() == lprobs.dim() - 1: |
| 50 | target = target.unsqueeze(-1) |
| 51 | nll_loss = -lprobs.gather(dim=-1, index=target) |
| 52 | smooth_loss = -lprobs.sum(dim=-1, keepdim=True) |
| 53 | if ignore_index is not None: |
| 54 | pad_mask = target.eq(ignore_index) |
| 55 | nll_loss.masked_fill_(pad_mask, 0.0) |
| 56 | smooth_loss.masked_fill_(pad_mask, 0.0) |
| 57 | else: |
| 58 | nll_loss = nll_loss.squeeze(-1) |
| 59 | smooth_loss = smooth_loss.squeeze(-1) |
| 60 | |
| 61 | nll_loss = nll_loss.sum() # mean()? Scared to break other math. |
| 62 | smooth_loss = smooth_loss.sum() |
| 63 | eps_i = epsilon / lprobs.size(-1) |
| 64 | loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss |
| 65 | return loss, nll_loss |
| 66 | |
| 67 | |
| 68 | def lmap(f: Callable, x: Iterable) -> List: |
nothing calls this directly
no outgoing calls
no test coverage detected