Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory. Args: vocab_path (:obj:`str`): The directory in which to save the vocabulary. Returns: :obj:`Tuple(str)`: Paths to the files saved.
(self, vocab_path)
| 317 | return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] |
| 318 | |
| 319 | def save_vocabulary(self, vocab_path): |
| 320 | """ |
| 321 | Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory. |
| 322 | |
| 323 | Args: |
| 324 | vocab_path (:obj:`str`): |
| 325 | The directory in which to save the vocabulary. |
| 326 | |
| 327 | Returns: |
| 328 | :obj:`Tuple(str)`: Paths to the files saved. |
| 329 | """ |
| 330 | index = 0 |
| 331 | if os.path.isdir(vocab_path): |
| 332 | vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES["vocab_file"]) |
| 333 | else: |
| 334 | vocab_file = vocab_path |
| 335 | with open(vocab_file, "w", encoding="utf-8") as writer: |
| 336 | for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): |
| 337 | if index != token_index: |
| 338 | logger.warning( |
| 339 | "Saving vocabulary to {}: vocabulary indices are not consecutive." |
| 340 | " Please check that the vocabulary is not corrupted!".format(vocab_file) |
| 341 | ) |
| 342 | index = token_index |
| 343 | writer.write(token + "\n") |
| 344 | index += 1 |
| 345 | return (vocab_file,) |
| 346 | |
| 347 | |
| 348 | class BasicTokenizer(object): |
no test coverage detected