| 152 | self.load_clip_model() |
| 153 | |
| 154 | def load_clip_model(self): |
| 155 | from transformers import AutoModel, AutoProcessor |
| 156 | |
| 157 | with open(os.path.join(self.model_path, "config.json")) as f: |
| 158 | config = json.load(f) |
| 159 | model_type = config.get("architectures", [None])[0] |
| 160 | self.model_type = model_type |
| 161 | |
| 162 | if model_type == "CLIPModel" or model_type == "ChineseCLIPModel": |
| 163 | self.model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True) |
| 164 | self.processor = AutoProcessor.from_pretrained(self.model_path, trust_remote_code=True) |
| 165 | # set model max length for chineseclipmodel |
| 166 | elif model_type.endswith("CLIPModel"): |
| 167 | self.model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True) |
| 168 | self.processor = None |
| 169 | else: |
| 170 | raise NotImplementedError(f"Unsupported model type: {model_type}") |
| 171 | |
| 172 | self.model.eval() |
| 173 | self.model.cuda() |
| 174 | |
| 175 | # set model max length for model that not specified in config.json |
| 176 | if self.processor is not None and self.processor.tokenizer.model_max_length > 100000: |
| 177 | try: |
| 178 | model_max_length = config['text_config']['max_position_embeddings'] |
| 179 | except: |
| 180 | model_max_length = 512 |
| 181 | self.processor.tokenizer.model_max_length = model_max_length |
| 182 | |
| 183 | |
| 184 | @torch.inference_mode() |