(self)
| 563 | """Tests preprocess_chunks.""" |
| 564 | |
| 565 | def test_preprocess_chunks(self): |
| 566 | texts = [ |
| 567 | "x" * 10, |
| 568 | "y" * 15, |
| 569 | ] |
| 570 | ds_fn = make_ds_fn(False, texts, repeat=1) |
| 571 | ds = ds_fn() |
| 572 | processed_ds = input_text.preprocess_chunks(chunk_size=4)(ds) |
| 573 | expected = [ |
| 574 | "xxxx", |
| 575 | "xxxx", |
| 576 | "xx", |
| 577 | "yyyy", |
| 578 | "yyyy", |
| 579 | "yyyy", |
| 580 | "yyy", |
| 581 | ] |
| 582 | actual = [extract_text(x) for x in processed_ds] |
| 583 | self.assertEqual(expected, actual) |
| 584 | |
| 585 | processed_ds = input_text.preprocess_chunks(chunk_size=5)(ds) |
| 586 | expected = [ |
| 587 | "xxxxx", |
| 588 | "xxxxx", |
| 589 | "yyyyy", |
| 590 | "yyyyy", |
| 591 | "yyyyy", |
| 592 | ] |
| 593 | actual = [extract_text(x) for x in processed_ds] |
| 594 | self.assertEqual(expected, actual) |
| 595 | |
| 596 | processed_ds = input_text.preprocess_chunks(chunk_size=100)(ds) |
| 597 | expected = texts |
| 598 | actual = [extract_text(x) for x in processed_ds] |
| 599 | self.assertEqual(expected, actual) |
| 600 | |
| 601 | |
| 602 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected