tokenize lambada dataset
(tokenizer, file_path, seq_length, eot)
| 122 | |
| 123 | |
| 124 | def tokenize_lambada(tokenizer, file_path, seq_length, eot): |
| 125 | """tokenize lambada dataset""" |
| 126 | content = [] |
| 127 | with open(file_path, "r", encoding="utf-8") as f: |
| 128 | for line in f.readlines(): |
| 129 | para = ( |
| 130 | json.loads(line)["text"] |
| 131 | .replace("“", '"') |
| 132 | .replace("”", '"') |
| 133 | .strip() |
| 134 | .strip(".") |
| 135 | ) |
| 136 | tokenized_text = tokenizer.tokenize(para) |
| 137 | content += tokenizer.convert_tokens_to_ids(tokenized_text) + [eot] |
| 138 | for chunk in chunks(content, seq_length): |
| 139 | sample = {} |
| 140 | if len(chunk) == seq_length: |
| 141 | sample["input_ids"] = np.array(chunk, dtype=np.int32) |
| 142 | yield sample |
| 143 | |
| 144 | |
| 145 | def task_unit(iterator, tokenizer, seq_length, eot, parallel_writer=True): |
no test coverage detected