| 502 | class LazySupervisedDataset(Dataset): |
| 503 | |
| 504 | def __init__(self, data_path: str, |
| 505 | tokenizer: transformers.PreTrainedTokenizer, |
| 506 | multimodal_cfg: dict): |
| 507 | super(LazySupervisedDataset, self).__init__() |
| 508 | |
| 509 | self.tokenizer, self.multimodal_cfg = tokenizer, multimodal_cfg |
| 510 | |
| 511 | self.pkl, self.prompt = pickle.load(open('./_data/ipr2pr.pkl', 'rb'))['task'], json.load(open('./_data/ipr2pr_expressive.json', 'r')) |
| 512 | random.shuffle(self.pkl) |
| 513 | print('--pkl: %d--'%(len(self.pkl))) |
| 514 | |
| 515 | def __len__(self): |
| 516 | return len(self.pkl) |