(example_batch, tokenizer)
| 71 | |
| 72 | @staticmethod |
| 73 | def convert_to_features(example_batch, tokenizer): |
| 74 | q1 = tokenizer.batch_encode_plus(example_batch['src'], max_length=128, truncation=True, add_special_tokens=False) |
| 75 | q2 = tokenizer.batch_encode_plus(example_batch['trg'], max_length=128, truncation=True, add_special_tokens=False) |
| 76 | encodings = { |
| 77 | 'source': q1['input_ids'], |
| 78 | 'target': q2['input_ids'], |
| 79 | } |
| 80 | |
| 81 | return encodings |
| 82 | |
| 83 | @staticmethod |
| 84 | def collate_fn(batch_input, tokenizer): |
nothing calls this directly
no outgoing calls
no test coverage detected