| 161 | self.annotators = set() |
| 162 | |
| 163 | def tokenize(self, text): |
| 164 | data = [] |
| 165 | matches = [m for m in self._regexp.finditer(text)] |
| 166 | for i in range(len(matches)): |
| 167 | # Get text |
| 168 | token = matches[i].group() |
| 169 | |
| 170 | # Get whitespace |
| 171 | span = matches[i].span() |
| 172 | start_ws = span[0] |
| 173 | if i + 1 < len(matches): |
| 174 | end_ws = matches[i + 1].span()[0] |
| 175 | else: |
| 176 | end_ws = span[1] |
| 177 | |
| 178 | # Format data |
| 179 | data.append( |
| 180 | ( |
| 181 | token, |
| 182 | text[start_ws:end_ws], |
| 183 | span, |
| 184 | ) |
| 185 | ) |
| 186 | return Tokens(data, self.annotators) |