| 222 | |
| 223 | |
| 224 | class SD3Tokenizer: |
| 225 | def __init__(self): |
| 226 | clip_tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") |
| 227 | self.clip_l = SDTokenizer(tokenizer=clip_tokenizer) |
| 228 | self.clip_g = SDXLClipGTokenizer(clip_tokenizer) |
| 229 | self.t5xxl = T5XXLTokenizer() |
| 230 | |
| 231 | def tokenize_with_weights(self, text:str): |
| 232 | out = {} |
| 233 | out["g"] = self.clip_g.tokenize_with_weights(text) |
| 234 | out["l"] = self.clip_l.tokenize_with_weights(text) |
| 235 | out["t5xxl"] = self.t5xxl.tokenize_with_weights(text) |
| 236 | return out |
| 237 | |
| 238 | |
| 239 | class ClipTokenWeightEncoder: |