(cls)
| 111 | |
| 112 | @classmethod |
| 113 | def setUpClass(cls): |
| 114 | source = "gs://maxtext-gemma/huggingface/gemma2-2b" |
| 115 | destination = os.path.join(MAXTEXT_ASSETS_ROOT, "") |
| 116 | subprocess.run( |
| 117 | ["gcloud", "storage", "cp", "-R", source, destination], |
| 118 | check=True, |
| 119 | ) |
| 120 | cls.hf_tokenizer = _input_pipeline_utils.get_tokenizer( |
| 121 | os.path.join(MAXTEXT_ASSETS_ROOT, "gemma2-2b"), "huggingface", add_bos=False, add_eos=False |
| 122 | ) |
| 123 | cls.sp_tokenizer = _input_pipeline_utils.get_tokenizer( |
| 124 | os.path.join(MAXTEXT_ASSETS_ROOT, "tokenizer.gemma"), "sentencepiece", add_bos=False, add_eos=False |
| 125 | ) |
| 126 | |
| 127 | @pytest.mark.tpu_only |
| 128 | def test_tokenize(self): |
nothing calls this directly
no test coverage detected