Class EvaluatingDataset

utils/dataset_utils.py:191–212

class EvaluatingDataset(torch.utils.data.Dataset):
    
    def __init__(self, dataset, tokenizer, label_name, valid_length=4096):
                
        self.dataset = dataset
        self.tokenizer = tokenizer
        self.label_name = label_name
        
        self.len = len(dataset)
        self.valid_length = valid_length 
        
    def __len__(self):
        return self.len
    
    def __getitem__(self, idx):

        sample = self.dataset[idx]
        inputs = self.tokenizer(sample[self.label_name], truncation=True, 
                                   max_length=self.valid_length, padding='max_length')
        inputs.update({'labels': inputs['input_ids']})

        return inputs