(data_prefix, data_impl, skip_warmup)
| 427 | |
| 428 | |
| 429 | def get_indexed_dataset_(data_prefix, data_impl, skip_warmup): |
| 430 | |
| 431 | print_rank_0(" > building dataset index ...") |
| 432 | |
| 433 | start_time = time.time() |
| 434 | indexed_dataset = make_indexed_dataset(data_prefix, data_impl, skip_warmup) |
| 435 | assert indexed_dataset.sizes.shape[0] == indexed_dataset.doc_idx[-1] |
| 436 | print_rank_0( |
| 437 | " > finished creating indexed dataset in {:4f} " |
| 438 | "seconds".format(time.time() - start_time) |
| 439 | ) |
| 440 | |
| 441 | print_rank_0(" > indexed dataset stats:") |
| 442 | print_rank_0( |
| 443 | " number of documents: {}".format(indexed_dataset.doc_idx.shape[0] - 1) |
| 444 | ) |
| 445 | print_rank_0(" number of sentences: {}".format(indexed_dataset.sizes.shape[0])) |
| 446 | |
| 447 | return indexed_dataset |
| 448 | |
| 449 | |
| 450 | def get_train_valid_test_split_(splits_string, size): |
nothing calls this directly
no test coverage detected