Tokenizes a piece of text.
(self, text)
| 190 | self.do_lower_case = do_lower_case |
| 191 | |
| 192 | def tokenize(self, text): |
| 193 | """Tokenizes a piece of text.""" |
| 194 | text = convert_to_unicode(text) |
| 195 | text = self._clean_text(text) |
| 196 | |
| 197 | # This was added on November 1st, 2018 for the multilingual and Chinese |
| 198 | # models. This is also applied to the English models now, but it doesn't |
| 199 | # matter since the English models were not trained on any Chinese data |
| 200 | # and generally don't have any Chinese data in them (there are Chinese |
| 201 | # characters in the vocabulary because Wikipedia does have some Chinese |
| 202 | # words in the English Wikipedia.). |
| 203 | text = self._tokenize_chinese_chars(text) |
| 204 | |
| 205 | orig_tokens = whitespace_tokenize(text) |
| 206 | split_tokens = [] |
| 207 | for token in orig_tokens: |
| 208 | if self.do_lower_case: |
| 209 | token = token.lower() |
| 210 | token = self._run_strip_accents(token) |
| 211 | split_tokens.extend(self._run_split_on_punc(token)) |
| 212 | |
| 213 | output_tokens = whitespace_tokenize(" ".join(split_tokens)) |
| 214 | return output_tokens |
| 215 | |
| 216 | def _run_strip_accents(self, text): |
| 217 | """Strips accents from a piece of text.""" |
no test coverage detected