(self, datapaths, tokenizer, max_source_seq_length,
max_target_seq_length)
| 127 | class GLMRawDataset(torch.utils.data.Dataset): |
| 128 | """GLM dataset class.""" |
| 129 | def __init__(self, datapaths, tokenizer, max_source_seq_length, |
| 130 | max_target_seq_length): |
| 131 | self.tokenizer = tokenizer |
| 132 | self.prompt = '' |
| 133 | self.samples = [] |
| 134 | self.random = random.Random(1234) |
| 135 | self.blank_maskratio = 0.1 |
| 136 | self.max_src_length, self.max_tgt_length =\ |
| 137 | max_source_seq_length, max_target_seq_length |
| 138 | for datapath in datapaths: |
| 139 | self.samples.extend( |
| 140 | self.process_samples_from_single_path(datapath)) |
| 141 | print(' >> total number of samples: {}'.format(len(self.samples))) |
| 142 | |
| 143 | def __len__(self): |
| 144 | return len(self.samples) |
nothing calls this directly
no test coverage detected