| 15 | |
| 16 | @staticmethod |
| 17 | def load_single(src_path, tgt_path, src_lang, tgt_lang): |
| 18 | |
| 19 | with open(src_path, 'r', encoding='utf-8') as f: |
| 20 | src_lines = f.readlines() |
| 21 | with open(tgt_path, 'r', encoding='utf-8') as f: |
| 22 | tgt_lines = f.readlines() |
| 23 | assert len(src_lines) == len(tgt_lines) |
| 24 | dataset_list = [{ |
| 25 | f'sentence_{src_lang}': src_lines[i].strip(), |
| 26 | f'sentence_{tgt_lang}': tgt_lines[i].strip(), |
| 27 | } for i in range(len(src_lines))] |
| 28 | return Dataset.from_list(dataset_list) |
| 29 | |
| 30 | @staticmethod |
| 31 | def load(path, name): |