(argv)
| 141 | |
| 142 | |
| 143 | def main(argv): |
| 144 | del argv |
| 145 | os.environ["TFDS_DATA_DIR"] = _DATASET_PATH.value |
| 146 | |
| 147 | read_config = tfds.ReadConfig( |
| 148 | shuffle_seed=0, |
| 149 | ) |
| 150 | train_ds_builder = tfds.builder(_DATASET_NAME.value) |
| 151 | train_ds = train_ds_builder.as_dataset(split="train", read_config=read_config, shuffle_files=True) |
| 152 | train_tokenizer( |
| 153 | train_ds, |
| 154 | vocab_path=os.path.join(_ASSETS_PATH.value, _VOCAB_MODEL_NAME.value), |
| 155 | vocab_size=_VOCAB_SIZE.value, |
| 156 | max_corpus_chars=_MAX_CORPUS_CHARS.value, |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected