(self, example, tokenizer)
| 62 | return dataset |
| 63 | |
| 64 | def add_original_src_length(self, example, tokenizer): |
| 65 | return { |
| 66 | 'original_src_length': len(tokenizer.encode(example['src'], max_length=128, truncation=True, add_special_tokens=False)) |
| 67 | } |
| 68 | |
| 69 | def my_load(self, splits): |
| 70 | return [self._load(name) for name in splits] |
nothing calls this directly
no outgoing calls
no test coverage detected