(self, vocab_path, do_lower_case)
| 413 | """Tokenizes text.""" |
| 414 | |
| 415 | def __init__(self, vocab_path, do_lower_case): |
| 416 | if isinstance(vocab_path, bytes): |
| 417 | vocab_path = vocab_path.decode() |
| 418 | |
| 419 | self.vocab_path = vocab_path |
| 420 | self.do_lower_case = do_lower_case |
| 421 | |
| 422 | self._base_tokenizer = tokenization.FullTokenizer( |
| 423 | vocab_file=vocab_path, do_lower_case=do_lower_case) |
| 424 | |
| 425 | # Look up special tokens. |
| 426 | self.cls_id, self.sep_id, self.mask_id = ( |
| 427 | self._base_tokenizer.convert_tokens_to_ids(['[CLS]', '[SEP]', |
| 428 | '[MASK]'])) |
| 429 | |
| 430 | def tokenize(self, text, compute_token_boundaries=True): |
| 431 | """Returns a list of Tokens.""" |
nothing calls this directly
no test coverage detected