(self, normalizer: InstantiableConfig, expected: list[str])
| 498 | ), |
| 499 | ) |
| 500 | def test_normalize(self, normalizer: InstantiableConfig, expected: list[str]): |
| 501 | texts = ["ah\u535a\u63a8zz \tHeLLo!how \n Are yoU? "] |
| 502 | ds_fn = make_ds_fn(False, texts, repeat=1) |
| 503 | process_fn = normalizer.set(input_key="text").instantiate() |
| 504 | processed_ds = process_fn(ds_fn()) |
| 505 | self.assertEqual(expected, [extract_text(x) for x in processed_ds]) |
| 506 | # Ensure that other fields are not dropped. |
| 507 | self.assertTrue(all("index" in x for x in processed_ds)) |
| 508 | # Test with multiple input keys. |
| 509 | ds_fn = make_seq2seq_ds_fn(False, texts, texts, repeat=1) |
| 510 | process_fn = normalizer.set(input_key=["source", "target"]).instantiate() |
| 511 | processed_ds = process_fn(ds_fn()) |
| 512 | for key in ["source", "target"]: |
| 513 | self.assertEqual(expected, [extract_text(x, input_key=key) for x in processed_ds]) |
| 514 | |
| 515 | @parameterized.parameters( |
| 516 | dict( |
nothing calls this directly
no test coverage detected