(self, train_file, sample=-1, seed=0, max_len=2048, category="", dedup=False, tokenizer=None, test=False)
| 86 | |
| 87 | class CSVBaseDataset(BaseDataset): |
| 88 | def __init__(self, train_file, sample=-1, seed=0, max_len=2048, category="", dedup=False, tokenizer=None, test=False): |
| 89 | super().__init__(tokenizer, max_len, test, category, dedup, seed) |
| 90 | |
| 91 | self.data = pd.read_csv(train_file) |
| 92 | |
| 93 | if sample > 0: |
| 94 | self.data = self.data.sample(sample, random_state=seed) |
| 95 | |
| 96 | |
| 97 | class JSONBaseDataset(BaseDataset): |