(path, args)
| 103 | return self.process_fn(super().__getitem__(idx)) |
| 104 | |
| 105 | def create_dataset_function(path, args): |
| 106 | num_samples = 100000 |
| 107 | num_features = 5 |
| 108 | |
| 109 | x = torch.randint(0, 100, (num_samples, num_features)) |
| 110 | |
| 111 | def process_fn(row): |
| 112 | return {'text': row[0], |
| 113 | 'loss_mask': np.array([1]*len(row[0])) |
| 114 | } |
| 115 | return FakeDataset(process_fn, x) |
| 116 | |
| 117 | if __name__ == '__main__': |
| 118 | import os |
no test coverage detected