Method get_tokenizer

llmware/models.py:12237–12265

Retrieves and instantiates a tokenizer. Takes a model_name as input, which is assumed to map to the Huggingface repository name - this name is not necessarily the same as the LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here

(self, model_name, **kwargs)

        return model

    def get_tokenizer(self, model_name, **kwargs):

        """ Retrieves and instantiates a tokenizer.  Takes a model_name as input, which is
        assumed to map to the Huggingface repository name - this name is not necessarily the same as the
        LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here
        should be the hf_repo attribute on the model card. """

        tokenizer = None

        self.model_name = model_name

        if self.custom_loader:
            tokenizer = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code,
                                                  self.custom_loader, caller="tokenizer", **kwargs)
        else:

            try:
                # will wrap in Exception if import fails
                from transformers import AutoTokenizer
            except ImportError:
                raise DependencyNotInstalledException("transformers")

            if self.api_key:
                tokenizer = AutoTokenizer.from_pretrained(model_name, token=self.api_key,
                                                          trust_remote_code=self.trust_remote_code)
            else:
                tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=self.trust_remote_code)

        return tokenizer


class CustomPTLoader: