(tokenizer, args)
| 170 | |
| 171 | |
| 172 | def build_wikitext103_dataset(tokenizer, args): |
| 173 | """""" |
| 174 | |
| 175 | assert len(args.valid_data) == 1 |
| 176 | with open(args.valid_data[0], "rb") as reader: |
| 177 | entire_data = reader.read().decode('utf-8') |
| 178 | num_original_tokens = len(entire_data.strip().split(" ")) |
| 179 | entire_data = get_detokenizer('wikitext')(entire_data) |
| 180 | print_rank_0(entire_data[:1024]) |
| 181 | tokenized_data = tokenizer.EncodeAsIds(entire_data).tokenization |
| 182 | num_tokenized_tokens = len(tokenized_data) |
| 183 | |
| 184 | val_dataset = LMDataset(args, [tokenized_data], tokenizer, num_original_tokens, num_tokenized_tokens) |
| 185 | print_rank_0(' > number of original tokens: {}, number of detokenized ' |
| 186 | 'tokens: {}'.format(num_original_tokens, num_tokenized_tokens)) |
| 187 | return val_dataset |
no test coverage detected