(self, index)
| 98 | return len(self.data) |
| 99 | |
| 100 | def __getitem__(self, index): |
| 101 | data = copy.deepcopy(self.data[index]) |
| 102 | no_loss_spans = copy.deepcopy(self.no_loss_spans[index]) |
| 103 | |
| 104 | data = torch.tensor(data, dtype=torch.long) |
| 105 | attn_mask = torch.ones_like(data, dtype=torch.bool) |
| 106 | label = copy.deepcopy(data) |
| 107 | |
| 108 | for no_loss_span in no_loss_spans: |
| 109 | label[no_loss_span[0] : no_loss_span[1]] = -100 |
| 110 | |
| 111 | return data, attn_mask, label |
| 112 | |
| 113 | def collate_fn(self, batch): |
| 114 | batch_input_ids, batch_attn_mask, batch_labels = [], [], [] |
nothing calls this directly
no outgoing calls
no test coverage detected