(cls)
| 78 | |
| 79 | @classmethod |
| 80 | def setUpClass(cls): |
| 81 | dataset_name = "c4/en:3.0.1" |
| 82 | dataset_path = "gs://maxtext-dataset" |
| 83 | cls.source_tokenizer = _input_pipeline_utils.get_tokenizer( |
| 84 | os.path.join(MAXTEXT_ASSETS_ROOT, "tokenizer_llama3.tiktoken"), |
| 85 | "tiktoken", |
| 86 | add_bos=False, |
| 87 | add_eos=False, |
| 88 | ) |
| 89 | os.environ["TFDS_DATA_DIR"] = dataset_path |
| 90 | read_config = tfds.ReadConfig( |
| 91 | shuffle_seed=0, |
| 92 | ) |
| 93 | train_ds_builder = tfds.builder(dataset_name) |
| 94 | cls.dataset = train_ds_builder.as_dataset(split="train", read_config=read_config, shuffle_files=True) |
| 95 | |
| 96 | @pytest.mark.tpu_only |
| 97 | def test_tokenize(self): |
nothing calls this directly
no test coverage detected