(self, max_concat_length, tokenizer, filename)
| 27 | |
| 28 | class Retrieval_Dataset(Dataset): |
| 29 | def __init__(self, max_concat_length, tokenizer, filename): |
| 30 | self.examples = [] |
| 31 | with open(filename, encoding="utf-8") as f: |
| 32 | data = f.readlines() |
| 33 | n = len(data) |
| 34 | |
| 35 | for i in range(n): |
| 36 | record = json.loads(data[i]) |
| 37 | sample_id = record['qid'] |
| 38 | rewite = record['rewrite'] |
| 39 | # rewite = record['truth_rewrite'] |
| 40 | rewrite_encoded = tokenizer.encode(rewite, add_special_tokens=True) |
| 41 | rewrite_padded, rewrite_mask = padding_seq_to_same_length(rewrite_encoded, max_pad_length=max_concat_length) |
| 42 | self.examples.append([sample_id, rewrite_padded, rewrite_mask]) |
| 43 | |
| 44 | def __len__(self): |
| 45 | return len(self.examples) |
nothing calls this directly
no test coverage detected