Method test_tokenizer

tests/test_tensorflow_text.py:7–12

(self)

class TestTensorflowText(unittest.TestCase):
    def test_tokenizer(self):
        word_tokenizer = tf_text.WhitespaceTokenizer()

        tokens = word_tokenizer.tokenize("I love Kaggle!")

        self.assertEqual((3,), tokens.shape)