Returns a list of Tokens.
(self, text, compute_token_boundaries=True)
| 428 | '[MASK]'])) |
| 429 | |
| 430 | def tokenize(self, text, compute_token_boundaries=True): |
| 431 | """Returns a list of Tokens.""" |
| 432 | token_strs = self._base_tokenizer.tokenize(text) |
| 433 | |
| 434 | if compute_token_boundaries: |
| 435 | # This runs in about half the time it takes for tokenization. |
| 436 | token_boundaries = self._compute_token_boundaries(text, token_strs) |
| 437 | else: |
| 438 | token_boundaries = [(-1, -1)] * len(token_strs) |
| 439 | |
| 440 | token_ids = self._base_tokenizer.convert_tokens_to_ids(token_strs) |
| 441 | |
| 442 | tokens = [] |
| 443 | for text, tid, (start, stop) in zip(token_strs, token_ids, |
| 444 | token_boundaries): |
| 445 | tokens.append(Token(text, start, stop, tid)) |
| 446 | return tokens |
| 447 | |
| 448 | def token_ids_to_str(self, token_ids): |
| 449 | """Converts a list of token IDs back to a human-readable string. |
no test coverage detected