(tokenizer, args)
| 154 | |
| 155 | |
| 156 | def build_lm_dataset(tokenizer, args): |
| 157 | documents = [] |
| 158 | num_tokens, num_original_tokens = 0, 0 |
| 159 | with open(args.valid_data[0], encoding='utf-8') as file: |
| 160 | for line in file: |
| 161 | tokens = tokenizer.EncodeAsIds(line.strip()).tokenization |
| 162 | num_tokens += len(tokens) |
| 163 | num_original_tokens += len(line.strip().split(" ")) |
| 164 | documents.append(tokens) |
| 165 | val_dataset = LMDataset(args, documents, tokenizer, num_original_tokens, num_tokens) |
| 166 | print_rank_0( |
| 167 | ' > number of document: {}, number of original tokens {}, number of detokenized tokens: {}'.format( |
| 168 | len(documents), num_original_tokens, num_tokens)) |
| 169 | return val_dataset |
| 170 | |
| 171 | |
| 172 | def build_wikitext103_dataset(tokenizer, args): |
no test coverage detected