Tokenizes a piece of text.
(self, text)
| 241 | self.do_lower_case = do_lower_case |
| 242 | |
| 243 | def tokenize(self, text): |
| 244 | """Tokenizes a piece of text.""" |
| 245 | text = convert_to_unicode(text) |
| 246 | text = self._clean_text(text) |
| 247 | |
| 248 | # This was added on November 1st, 2018 for the multilingual and Chinese |
| 249 | # models. This is also applied to the English models now, but it doesn't |
| 250 | # matter since the English models were not trained on any Chinese data |
| 251 | # and generally don't have any Chinese data in them (there are Chinese |
| 252 | # characters in the vocabulary because Wikipedia does have some Chinese |
| 253 | # words in the English Wikipedia.). |
| 254 | text = self._tokenize_chinese_chars(text) |
| 255 | |
| 256 | orig_tokens = whitespace_tokenize(text) |
| 257 | split_tokens = [] |
| 258 | for token in orig_tokens: |
| 259 | if self.do_lower_case: |
| 260 | token = token.lower() |
| 261 | token = self._run_strip_accents(token) |
| 262 | split_tokens.extend(self._run_split_on_punc(token)) |
| 263 | |
| 264 | output_tokens = whitespace_tokenize(" ".join(split_tokens)) |
| 265 | return output_tokens |
| 266 | |
| 267 | def _run_strip_accents(self, text): |
| 268 | """Strips accents from a piece of text.""" |
no test coverage detected