(self, split)
| 64 | self.tokenizer = tokenizer |
| 65 | |
| 66 | def create_examples(self, split): |
| 67 | if split == "train": |
| 68 | filename = "train" |
| 69 | elif split == "dev": |
| 70 | filename = "val" |
| 71 | elif split == "test": |
| 72 | filename = "test" |
| 73 | else: |
| 74 | raise NotImplementedError(split) |
| 75 | print_rank_0(f"Creating {self.task}-{split} dataset from {self.data_dir}") |
| 76 | if self.task == "gigaword": |
| 77 | detokenizer = gigaword_detokenize |
| 78 | elif self.task == "cnn_dm": |
| 79 | detokenizer = cnndm_detokenize |
| 80 | else: |
| 81 | detokenizer = None |
| 82 | source_texts, target_texts = [], [] |
| 83 | with open(os.path.join(self.data_dir, f"{filename}.source"), encoding='utf-8') as file: |
| 84 | for line in file: |
| 85 | line = line.strip() |
| 86 | line = punctuation_standardization(line) |
| 87 | line = detokenizer(line) if detokenizer else line |
| 88 | source_texts.append(line) |
| 89 | with open(os.path.join(self.data_dir, f"{filename}.target"), encoding='utf-8') as file: |
| 90 | for line in file: |
| 91 | line = line.strip() |
| 92 | line = punctuation_standardization(line) |
| 93 | line = detokenizer(line, is_target=True) if detokenizer else line |
| 94 | target_texts.append(line) |
| 95 | assert len(source_texts) == len(target_texts) |
| 96 | example_list = [] |
| 97 | for idx, (source_text, target_text) in enumerate(zip(source_texts, target_texts)): |
| 98 | if (idx + 1) % 20000 == 0: |
| 99 | print_rank_0(f"Complete {idx + 1} examples") |
| 100 | guid = "%s-%s" % (split, idx) |
| 101 | meta = {"ref": self.tokenizer.DecodeIds(self.tokenizer.EncodeAsIds(target_text).tokenization)} |
| 102 | example = InputExample(guid=guid, text_a=source_text, text_b=target_text, meta=meta) |
| 103 | if idx < 10: |
| 104 | print_rank_0((source_text.encode('utf-8'), target_text.encode('utf-8'), meta["ref"].encode('utf-8'))) |
| 105 | example_list.append(example) |
| 106 | return example_list |
| 107 | |
| 108 | class CMRCProcessor: |
| 109 | def __init__(self, data_dir, tokenizer): |
nothing calls this directly
no test coverage detected