| 632 | assert result == expected |
| 633 | |
| 634 | def test_ngrams2(self): |
| 635 | tokens = ''' |
| 636 | Redistribution and use in source and binary are permitted. |
| 637 | '''.split() |
| 638 | |
| 639 | result = list(ngrams(tokens, ngram_length=4)) |
| 640 | expected = [ |
| 641 | ('Redistribution', 'and', 'use', 'in'), |
| 642 | ('and', 'use', 'in', 'source'), |
| 643 | ('use', 'in', 'source', 'and'), |
| 644 | ('in', 'source', 'and', 'binary'), |
| 645 | ('source', 'and', 'binary', 'are'), |
| 646 | ('and', 'binary', 'are', 'permitted.')] |
| 647 | |
| 648 | assert result == expected |
| 649 | |
| 650 | def test_select_ngrams_with_unicode_inputs(self): |
| 651 | result = list(select_ngrams(x for x in [('b', 'ä', 'c'), ('ä', 'ä', 'c'), ('e', 'ä', 'c'), ('b', 'f', 'ä'), ('g', 'c', 'd')])) |