| 30 | return encodings |
| 31 | |
| 32 | class ConditionalLoader: |
| 33 | def __init__(self, tokenizer, return_source_length=False): |
| 34 | self.tokenizer = tokenizer |
| 35 | self.return_source_length = return_source_length |
| 36 | self.data_dir = './conditional_data' |
| 37 | |
| 38 | @staticmethod |
| 39 | def _convert_to_features_original(example_batch, tokenizer): |
| 40 | q1 = tokenizer.batch_encode_plus(example_batch['src'], max_length=128, truncation=True, add_special_tokens=False) |
| 41 | q2 = tokenizer.batch_encode_plus(example_batch['trg'], max_length=128, truncation=True, add_special_tokens=False) |
| 42 | return { |
| 43 | 'source': q1['input_ids'], |
| 44 | 'target': q2['input_ids'], |
| 45 | } |
| 46 | |
| 47 | def load_original(self, split): |
| 48 | dataset = datasets.load_dataset(os.path.join(self.data_dir, self.task_name, f'{self.task_name}.py'), split=split) |
| 49 | dataset = dataset.map(partial(self._convert_to_features_original, tokenizer=self.tokenizer), batched=True, load_from_cache_file=False) |
| 50 | print(f'Example in {split} set:') |
| 51 | print(dataset[0]) |
| 52 | return dataset |
| 53 | |
| 54 | def _load(self, split): |
| 55 | dataset = datasets.load_dataset(os.path.join(self.data_dir, self.task_name, f'{self.task_name}.py'), split=split) |
| 56 | if self.return_source_length: |
| 57 | dataset = dataset.map(partial(self.add_original_src_length, tokenizer=self.tokenizer)) |
| 58 | dataset = dataset.map(self.add_prompt) |
| 59 | dataset = dataset.map(partial(self.convert_to_features, tokenizer=self.tokenizer), batched=True) |
| 60 | print(f'Example in {split} set:') |
| 61 | print(dataset[0]) |
| 62 | return dataset |
| 63 | |
| 64 | def add_original_src_length(self, example, tokenizer): |
| 65 | return { |
| 66 | 'original_src_length': len(tokenizer.encode(example['src'], max_length=128, truncation=True, add_special_tokens=False)) |
| 67 | } |
| 68 | |
| 69 | def my_load(self, splits): |
| 70 | return [self._load(name) for name in splits] |
| 71 | |
| 72 | @staticmethod |
| 73 | def convert_to_features(example_batch, tokenizer): |
| 74 | q1 = tokenizer.batch_encode_plus(example_batch['src'], max_length=128, truncation=True, add_special_tokens=False) |
| 75 | q2 = tokenizer.batch_encode_plus(example_batch['trg'], max_length=128, truncation=True, add_special_tokens=False) |
| 76 | encodings = { |
| 77 | 'source': q1['input_ids'], |
| 78 | 'target': q2['input_ids'], |
| 79 | } |
| 80 | |
| 81 | return encodings |
| 82 | |
| 83 | @staticmethod |
| 84 | def collate_fn(batch_input, tokenizer): |
| 85 | input_ids = pad_sequence([torch.tensor( |
| 86 | [tokenizer.cls_token_id] + d['source'] + d['target'] + [tokenizer.sep_token_id] |
| 87 | ) for d in batch_input], batch_first=True) |
| 88 | |
| 89 | attention_mask = torch.ones_like(input_ids) |
nothing calls this directly
no outgoing calls
no test coverage detected