tokenize wikitext-2/wikitext-103 dataset
(tokenizer, file_path, seq_length, eot)
| 105 | |
| 106 | |
| 107 | def tokenize_wiki(tokenizer, file_path, seq_length, eot): |
| 108 | """tokenize wikitext-2/wikitext-103 dataset""" |
| 109 | content = [] |
| 110 | with open(file_path, "r", encoding="utf-8") as f: |
| 111 | for para in clean_wikitext(f.read()).split("\n\n"): |
| 112 | if para and para.strip().startswith("=") is False: |
| 113 | tokenized_text = tokenizer.tokenize(para) |
| 114 | content += tokenizer.convert_tokens_to_ids(tokenized_text) + [ |
| 115 | eot |
| 116 | ] |
| 117 | for chunk in chunks(content, seq_length): |
| 118 | sample = {} |
| 119 | if len(chunk) == seq_length: |
| 120 | sample["input_ids"] = np.array(chunk, dtype=np.int32) |
| 121 | yield sample |
| 122 | |
| 123 | |
| 124 | def tokenize_lambada(tokenizer, file_path, seq_length, eot): |
no test coverage detected