(self, *, vocab_cfg: InstantiableConfig, newlines_replaced_with: str)
| 236 | |
| 237 | class TokenizeExampleTest(test_utils.TestCase): |
| 238 | def _test_tokenize_example(self, *, vocab_cfg: InstantiableConfig, newlines_replaced_with: str): |
| 239 | vocab = vocab_cfg.instantiate() |
| 240 | newlines_replaced_with_id = vocab.encode(newlines_replaced_with).pop() |
| 241 | |
| 242 | # Test tokenize_example replaces newlines. |
| 243 | tokens = input_text.tokenize_example( |
| 244 | "Hello\n", sp_vocab=vocab, replace_newlines_with=newlines_replaced_with |
| 245 | ).numpy() |
| 246 | self.assertNestedAllClose( |
| 247 | np.array([*vocab.encode("Hello"), newlines_replaced_with_id]), tokens |
| 248 | ) |
| 249 | |
| 250 | @parameterized.parameters( |
| 251 | dict( |
no test coverage detected