(cls)
| 32 | |
| 33 | @classmethod |
| 34 | def setUpClass(cls): |
| 35 | dataset_name = "c4/en:3.0.1" |
| 36 | dataset_path = "gs://maxtext-dataset" |
| 37 | cls.vocab_size = 32_768 |
| 38 | cls.max_corpus_chars = 10_000_000 |
| 39 | assets_path = "tests" |
| 40 | vocab_model_name = "test_tokenizer" |
| 41 | cls.tokenizer_path = os.path.join(assets_path, vocab_model_name) |
| 42 | cls.source_tokenizer = _input_pipeline_utils.get_tokenizer( |
| 43 | os.path.join(MAXTEXT_ASSETS_ROOT, "tokenizer"), "sentencepiece", add_bos=False, add_eos=False |
| 44 | ) |
| 45 | os.environ["TFDS_DATA_DIR"] = dataset_path |
| 46 | read_config = tfds.ReadConfig( |
| 47 | shuffle_seed=0, |
| 48 | ) |
| 49 | train_ds_builder = tfds.builder(dataset_name) |
| 50 | cls.dataset = train_ds_builder.as_dataset(split="train", read_config=read_config, shuffle_files=True) |
| 51 | train_tokenizer.train_tokenizer( |
| 52 | cls.dataset, |
| 53 | vocab_path=cls.tokenizer_path, |
| 54 | vocab_size=cls.vocab_size, |
| 55 | max_corpus_chars=cls.max_corpus_chars, |
| 56 | ) |
| 57 | cls.test_tokenizer = _input_pipeline_utils.get_tokenizer( |
| 58 | cls.tokenizer_path, "sentencepiece", add_bos=False, add_eos=False |
| 59 | ) |
| 60 | |
| 61 | @classmethod |
| 62 | def tearDownClass(cls): |
nothing calls this directly
no test coverage detected