(out_file, impl, vocab_size=None)
| 53 | |
| 54 | |
| 55 | def make_builder(out_file, impl, vocab_size=None): |
| 56 | if impl == "mmap": |
| 57 | return MMapIndexedDatasetBuilder( |
| 58 | out_file, dtype=__best_fitting_dtype(vocab_size) |
| 59 | ) |
| 60 | else: |
| 61 | return IndexedDatasetBuilder(out_file) |
| 62 | |
| 63 | |
| 64 | def make_dataset(path, impl, skip_warmup=False): |
nothing calls this directly
no test coverage detected