tokenize openwebtext dataset
(tokenizer, iterator, seq_length, eot)
| 85 | |
| 86 | |
| 87 | def tokenize_openwebtext(tokenizer, iterator, seq_length, eot): |
| 88 | """tokenize openwebtext dataset""" |
| 89 | for file_path in iterator: |
| 90 | if os.path.getsize(file_path) == 0: |
| 91 | continue |
| 92 | content = [] |
| 93 | with open(file_path, "r", encoding="utf-8") as f: |
| 94 | for para in f.read().split("\n\n"): |
| 95 | if para: |
| 96 | tokenized_text = tokenizer.tokenize(para) |
| 97 | content += tokenizer.convert_tokens_to_ids( |
| 98 | tokenized_text |
| 99 | ) + [eot] |
| 100 | for chunk in chunks(content, seq_length): |
| 101 | sample = {} |
| 102 | if len(chunk) == seq_length: |
| 103 | sample["input_ids"] = np.array(chunk, dtype=np.int32) |
| 104 | yield sample |
| 105 | |
| 106 | |
| 107 | def tokenize_wiki(tokenizer, file_path, seq_length, eot): |
no test coverage detected