| 631 | return len(self._tokenizer.vocab) |
| 632 | |
| 633 | def save_to_file(self, filename_prefix): |
| 634 | vocab_file = f'{filename_prefix}.vocab' |
| 635 | tf.io.gfile.copy(self._tokenizer.vocab_file, vocab_file, overwrite=True) |
| 636 | with tf.io.gfile.GFile(f'{filename_prefix}.encoder', 'w') as f: |
| 637 | json.dump({ |
| 638 | 'vocab_file': vocab_file, |
| 639 | 'do_lower_case': self._tokenizer.do_lower_case, |
| 640 | 'max_seq_length': self._max_seq_length, |
| 641 | 'include_title': self._include_title, |
| 642 | 'include_sentence_id': self._include_sentence_id, |
| 643 | }, f) |
| 644 | |
| 645 | @classmethod |
| 646 | def load_from_file(cls, filename_prefix): |