(self, train_file, tokenizer, max_len=2048, sample=-1, test=False, seed=0, category="", K=4, dedup=False)
| 37 | |
| 38 | class SidSFTDataset(Dataset): |
| 39 | def __init__(self, train_file, tokenizer, max_len=2048, sample=-1, test=False, seed=0, category="", K=4, dedup=False): |
| 40 | self.data = pd.read_csv(train_file) |
| 41 | random.seed(seed) |
| 42 | |
| 43 | if sample > 0: |
| 44 | self.data = self.data.sample(sample, random_state=seed) |
| 45 | self.tokenizer = Tokenizer(tokenizer) |
| 46 | self.test = test |
| 47 | self.max_len = max_len |
| 48 | self.category = category |
| 49 | self.dedup = dedup |
| 50 | self.get_inputs() |
| 51 | |
| 52 | def __len__(self): |
| 53 | return len(self.data) |
nothing calls this directly
no test coverage detected