Method test_tokenize_example

axlearn/common/input_text_test.py:260–265

(self, vocab_cfg: InstantiableConfig, newlines_replaced_with: str)

        ),
    )
    def test_tokenize_example(self, vocab_cfg: InstantiableConfig, newlines_replaced_with: str):
        if not os.path.exists(_OPT_VOCAB_FILE):
            self.skipTest("Missing testdata.")
        self._test_tokenize_example(
            vocab_cfg=vocab_cfg, newlines_replaced_with=newlines_replaced_with
        )


class NumBytesTest(test_utils.TestCase):