(self, model_path: str, trust_remote_code: bool = False)
| 424 | """ |
| 425 | |
| 426 | def __init__(self, model_path: str, trust_remote_code: bool = False): |
| 427 | from transformers import AutoConfig, PretrainedConfig |
| 428 | try: |
| 429 | model_cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code) |
| 430 | except Exception as e: # noqa |
| 431 | model_cfg = PretrainedConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code) |
| 432 | is_gpt_oss = getattr(model_cfg, 'model_type', '') == 'gpt_oss' |
| 433 | from transformers.models.auto.tokenization_auto import get_tokenizer_config |
| 434 | tokenizer_config = get_tokenizer_config(model_path, trust_remote_code=trust_remote_code) |
| 435 | config_tokenizer_class = tokenizer_config.get('tokenizer_class') |
| 436 | if config_tokenizer_class == 'ChatGLM4Tokenizer': |
| 437 | self.model = ChatGLM4Tokenizer(model_path, trust_remote_code=trust_remote_code) |
| 438 | elif config_tokenizer_class == 'ChatGLMTokenizer': |
| 439 | self.model = ChatGLMTokenizer(model_path, trust_remote_code=trust_remote_code) |
| 440 | elif is_gpt_oss: |
| 441 | self.model = GptOssTokenizer(model_path, trust_remote_code=trust_remote_code) |
| 442 | else: |
| 443 | self.model = HuggingFaceTokenizer(model_path, trust_remote_code=trust_remote_code) |
| 444 | self.logger = get_logger('lmdeploy') |
| 445 | |
| 446 | @property |
| 447 | def vocab_size(self): |
no test coverage detected