(self, checkpoint_dir: Path)
| 11 | |
| 12 | class Tokenizer: |
| 13 | def __init__(self, checkpoint_dir: Path) -> None: |
| 14 | # some checkpoints have both files, `.model` takes precedence |
| 15 | # TODO: Deprecate SentencePieceProcessor. It behaves differently from tokenizer.json |
| 16 | # So it's best to avoid and reduce complexity. |
| 17 | # For example, see: https://github.com/google/sentencepiece/issues/667 |
| 18 | # if (vocabulary_path := checkpoint_dir / 'tokenizer.model').is_file(): |
| 19 | # print( |
| 20 | # f'Tokenizer class is initialised with `{vocabulary_path}` as this takes precedence.' |
| 21 | # ) |
| 22 | # print( |
| 23 | # 'If you have intended to use `tokenizer.json`, please remove `tokenizer.model`.' |
| 24 | # ) |
| 25 | # from sentencepiece import SentencePieceProcessor |
| 26 | |
| 27 | # self.processor = SentencePieceProcessor(model_file=str(vocabulary_path)) |
| 28 | # self.backend = SENTENCEPIECE |
| 29 | # self.bos_id = self.processor.bos_id() |
| 30 | # self.eos_id = self.processor.eos_id() |
| 31 | # self.pad_id = self.processor.pad_id() |
| 32 | |
| 33 | # self.processor.Decode |
| 34 | |
| 35 | if (vocabulary_path := checkpoint_dir / 'tokenizer.json').is_file(): |
| 36 | print(f'Tokenizer class is initalised with `{vocabulary_path}`.') |
| 37 | from tokenizers import Tokenizer as HFTokenizer |
| 38 | |
| 39 | self.processor: HFTokenizer = HFTokenizer.from_file(str(vocabulary_path)) |
| 40 | self.backend = HUGGINGFACE |
| 41 | |
| 42 | with open(checkpoint_dir / 'tokenizer_config.json') as fp: |
| 43 | config = json.load(fp) |
| 44 | |
| 45 | bos_token_config = config.get('bos_token', None) |
| 46 | bos_token = ( |
| 47 | bos_token_config |
| 48 | # bos_token_config in tokenizer_config can be a str or object with 'content' |
| 49 | if isinstance(bos_token_config, str) |
| 50 | else bos_token_config['content'] |
| 51 | ) |
| 52 | self.bos_id = ( |
| 53 | self.token_to_id(bos_token) if bos_token_config is not None else None |
| 54 | ) |
| 55 | |
| 56 | eos_token_config = config.get('eos_token', None) |
| 57 | eos_token = ( |
| 58 | eos_token_config |
| 59 | if isinstance(eos_token_config, str) |
| 60 | else eos_token_config['content'] |
| 61 | ) |
| 62 | self.eos_id = ( |
| 63 | self.token_to_id(eos_token) if eos_token_config is not None else None |
| 64 | ) |
| 65 | |
| 66 | pad_token_config = config.get('pad_token', None) |
| 67 | pad_token = ( |
| 68 | pad_token_config |
| 69 | if pad_token_config is None or isinstance(pad_token_config, str) |
| 70 | else pad_token_config['content'] |
nothing calls this directly
no test coverage detected