Group consecutive entity tokens with the same NER tag.
(self)
| 105 | return ngrams |
| 106 | |
| 107 | def entity_groups(self): |
| 108 | """Group consecutive entity tokens with the same NER tag.""" |
| 109 | entities = self.entities() |
| 110 | if not entities: |
| 111 | return None |
| 112 | non_ent = self.opts.get("non_ent", "O") |
| 113 | groups = [] |
| 114 | idx = 0 |
| 115 | while idx < len(entities): |
| 116 | ner_tag = entities[idx] |
| 117 | # Check for entity tag |
| 118 | if ner_tag != non_ent: |
| 119 | # Chomp the sequence |
| 120 | start = idx |
| 121 | while idx < len(entities) and entities[idx] == ner_tag: |
| 122 | idx += 1 |
| 123 | groups.append((self.slice(start, idx).untokenize(), ner_tag)) |
| 124 | else: |
| 125 | idx += 1 |
| 126 | return groups |
| 127 | |
| 128 | |
| 129 | class Tokenizer(object): |
nothing calls this directly
no test coverage detected