Return a tokenizer
(self, metadata: TokenizerParameters)
| 1501 | raise KeyError(f"Unsupported tokenizer type: {self.config.tokenizer_type}") from None |
| 1502 | |
| 1503 | def build_tokenizer(self, metadata: TokenizerParameters) -> tokenizer_api.Tokenizer: |
| 1504 | """Return a tokenizer""" |
| 1505 | if metadata.tokenizer_type == TokenizerType.tiktoken: |
| 1506 | return token_utils.TikToken(metadata) |
| 1507 | elif metadata.tokenizer_type == TokenizerType.sentencepiece: |
| 1508 | return token_utils.SentencePieceTokenizer(metadata) |
| 1509 | elif metadata.tokenizer_type == TokenizerType.huggingface: |
| 1510 | tokenizer_model = token_utils.HuggingFaceTokenizer(metadata) |
| 1511 | if tokenizer_model.tokenizer.pad_token_id is None: |
| 1512 | if tokenizer_model.tokenizer.unk_token_id is not None: |
| 1513 | tokenizer_model.tokenizer.pad_token_id = tokenizer_model.tokenizer.unk_token_id |
| 1514 | else: |
| 1515 | print(f"Warning: setting pad_token_id to eos_token_id:{tokenizer_model.tokenizer.eos_token_id}") |
| 1516 | tokenizer_model.tokenizer.pad_token_id = tokenizer_model.tokenizer.eos_token_id |
| 1517 | return tokenizer_model |
| 1518 | else: |
| 1519 | raise ValueError(f"Unsupported tokenizer type: {metadata.tokenizer_type}") |
| 1520 | |
| 1521 | def init_decode_state( |
| 1522 | self, |
no outgoing calls
no test coverage detected