Class BaseDataset

data.py:37–84

        return self.tokenizer.decode(t)

class BaseDataset(Dataset):
    def __init__(self, tokenizer=None, max_len=2048, test=False, category="", dedup=False, seed=None):
        super().__init__()
        self.data = None
        self.inputs = None
        
        if tokenizer is not None:
            self.tokenizer = Tokenizer(tokenizer)
        if seed is not None:
            random.seed(seed)
        
        self.test = test
        self.max_len = max_len
        self.category = category
        self.dedup = dedup

    def __len__(self):
        return len(self.data)

    def get_inputs(self):
        inputs = []
        for i in tqdm(range(len(self.data))):
            inputs.append(self.pre(i))
        self.inputs = inputs

    def get_all(self):
        temp = []
        for i in range(len(self.data)):
            temp.append(self.get_history(self.data.iloc[i]))
        return temp

    def get_inputs_list(self):
        return self.inputs

    def __getitem__(self, idx):
        return self.inputs[idx]

    def pre(self, idx):
        raise NotImplementedError(None)

    def get_history(self, row):
        raise {}
       
    def generate_prompt(self, data_point):
        return f"""### User Input: 
{data_point["input"]}

### Response:\n{data_point["output"]}"""


class CSVBaseDataset(BaseDataset):