Method save

accessory/model/tokenizer.py:117–122

(self, save_dir: str)

        return self.tokenizer.decode(t)

    def save(self, save_dir: str):
        if self.tokenizer_type == "transformers":
            self.tokenizer.save_pretrained(save_dir)
        else:
            with open(Path(save_dir)/"tokenizer.model", 'wb') as f:
                f.write(self.tokenizer.serialized_model_proto())

    @ property
    def n_words(self):