Function make_tokenizer

demo/Diffusion/models.py:445–448

(version, hf_token)

            max_batch_size=max_batch_size, embedding_dim=get_embedding_dim(version))

def make_tokenizer(version, hf_token):
    return CLIPTokenizer.from_pretrained(get_path(version),
            subfolder="tokenizer",
            use_auth_token=hf_token)