A class to represent a list of tokenized text.
| 7 | |
| 8 | |
| 9 | class Tokens(object): |
| 10 | """A class to represent a list of tokenized text.""" |
| 11 | |
| 12 | TEXT = 0 |
| 13 | TEXT_WS = 1 |
| 14 | SPAN = 2 |
| 15 | POS = 3 |
| 16 | LEMMA = 4 |
| 17 | NER = 5 |
| 18 | |
| 19 | def __init__(self, data, annotators, opts=None): |
| 20 | self.data = data |
| 21 | self.annotators = annotators |
| 22 | self.opts = opts or {} |
| 23 | |
| 24 | def __len__(self): |
| 25 | """The number of tokens.""" |
| 26 | return len(self.data) |
| 27 | |
| 28 | def slice(self, i=None, j=None): |
| 29 | """Return a view of the list of tokens from [i, j).""" |
| 30 | new_tokens = copy.copy(self) |
| 31 | new_tokens.data = self.data[i:j] |
| 32 | return new_tokens |
| 33 | |
| 34 | def untokenize(self): |
| 35 | """Returns the original text (with whitespace reinserted).""" |
| 36 | return "".join([t[self.TEXT_WS] for t in self.data]).strip() |
| 37 | |
| 38 | def words(self, uncased=False): |
| 39 | """Returns a list of the text of each token |
| 40 | |
| 41 | Args: |
| 42 | uncased: lower cases text |
| 43 | """ |
| 44 | if uncased: |
| 45 | return [t[self.TEXT].lower() for t in self.data] |
| 46 | else: |
| 47 | return [t[self.TEXT] for t in self.data] |
| 48 | |
| 49 | def offsets(self): |
| 50 | """Returns a list of [start, end) character offsets of each token.""" |
| 51 | return [t[self.SPAN] for t in self.data] |
| 52 | |
| 53 | def pos(self): |
| 54 | """Returns a list of part-of-speech tags of each token. |
| 55 | Returns None if this annotation was not included. |
| 56 | """ |
| 57 | if "pos" not in self.annotators: |
| 58 | return None |
| 59 | return [t[self.POS] for t in self.data] |
| 60 | |
| 61 | def lemmas(self): |
| 62 | """Returns a list of the lemmatized text of each token. |
| 63 | Returns None if this annotation was not included. |
| 64 | """ |
| 65 | if "lemma" not in self.annotators: |
| 66 | return None |