Returns a list of n-grams (tuples of n successive words) from the given string. Alternatively, you can supply a Text or Sentence object. With continuous=False, n-grams will not run over sentence markers (i.e., .!?). Punctuation marks are stripped from words.
(string, n=3, punctuation=PUNCTUATION, continuous=False)
| 62 | PUNCTUATION = ".,;:!?()[]{}`''\"@#$^&*+-|=~_" |
| 63 | |
| 64 | def ngrams(string, n=3, punctuation=PUNCTUATION, continuous=False): |
| 65 | """ Returns a list of n-grams (tuples of n successive words) from the given string. |
| 66 | Alternatively, you can supply a Text or Sentence object. |
| 67 | With continuous=False, n-grams will not run over sentence markers (i.e., .!?). |
| 68 | Punctuation marks are stripped from words. |
| 69 | """ |
| 70 | def strip_punctuation(s, punctuation=set(punctuation)): |
| 71 | return [w for w in s if (isinstance(w, Word) and w.string or w) not in punctuation] |
| 72 | if n <= 0: |
| 73 | return [] |
| 74 | if isinstance(string, basestring): |
| 75 | s = [strip_punctuation(s.split(" ")) for s in tokenize(string)] |
| 76 | if isinstance(string, Sentence): |
| 77 | s = [strip_punctuation(string)] |
| 78 | if isinstance(string, Text): |
| 79 | s = [strip_punctuation(s) for s in string] |
| 80 | if continuous: |
| 81 | s = [sum(s, [])] |
| 82 | g = [] |
| 83 | for s in s: |
| 84 | #s = [None] + s + [None] |
| 85 | g.extend([tuple(s[i:i+n]) for i in range(len(s)-n+1)]) |
| 86 | return g |
| 87 | |
| 88 | def pprint(string, token=[WORD, POS, CHUNK, PNP], column=4): |
| 89 | """ Pretty-prints the output of Parser.parse() as a table with outlined columns. |