| 589 | test_data_dir = TEST_DATA_DIR |
| 590 | |
| 591 | def test_ngrams(self): |
| 592 | tokens = ''' |
| 593 | Redistribution and use in source and binary are permitted. |
| 594 | '''.split() |
| 595 | |
| 596 | result = list(ngrams(tokens, ngram_length=4)) |
| 597 | expected = [ |
| 598 | ('Redistribution', 'and', 'use', 'in'), |
| 599 | ('and', 'use', 'in', 'source'), |
| 600 | ('use', 'in', 'source', 'and'), |
| 601 | ('in', 'source', 'and', 'binary'), |
| 602 | ('source', 'and', 'binary', 'are'), |
| 603 | ('and', 'binary', 'are', 'permitted.') |
| 604 | ] |
| 605 | assert result == expected |
| 606 | |
| 607 | def test_ngrams_with_None(self): |
| 608 | tokens = ['Redistribution', 'and', 'use', None, 'in', 'source', 'and', 'binary', 'are', None] |