Compute a CLIP embedding for either an image or text. Parameters ---------- input_data : str or PIL.Image.Image - If a string: treated as a file path to an image (if file exists) or as a text prompt. - If a PIL.Image.Image: treated as an image. model : CLIPModel
(input_data, model, processor, device='cuda', input_type=None)
| 272 | return np.array(embeddings) # shape: (N, D) |
| 273 | |
| 274 | def compute_clip_embedding(input_data, model, processor, device='cuda', input_type=None): |
| 275 | """ |
| 276 | Compute a CLIP embedding for either an image or text. |
| 277 | |
| 278 | Parameters |
| 279 | ---------- |
| 280 | input_data : str or PIL.Image.Image |
| 281 | - If a string: treated as a file path to an image (if file exists) or as a text prompt. |
| 282 | - If a PIL.Image.Image: treated as an image. |
| 283 | model : CLIPModel |
| 284 | The loaded CLIP model (e.g., from Hugging Face). |
| 285 | processor : CLIPProcessor |
| 286 | The corresponding CLIP processor for tokenization/preprocessing. |
| 287 | device : torch.device |
| 288 | The device to run inference on. |
| 289 | input_type : {'image', 'text', None}, optional |
| 290 | Force the mode; if `None` (default) the function will try to infer from `input_data`. |
| 291 | |
| 292 | Returns |
| 293 | ------- |
| 294 | np.ndarray |
| 295 | A 1D NumPy array of length D (the CLIP embedding dimension). |
| 296 | """ |
| 297 | model.eval() |
| 298 | |
| 299 | # Decide mode |
| 300 | if input_type == "image": |
| 301 | mode = "image" |
| 302 | elif input_type == "text": |
| 303 | mode = "text" |
| 304 | else: |
| 305 | # auto-detect |
| 306 | if isinstance(input_data, Image.Image): |
| 307 | mode = "image" |
| 308 | elif isinstance(input_data, str) and os.path.isfile(input_data): |
| 309 | mode = "image" |
| 310 | else: |
| 311 | mode = "text" |
| 312 | |
| 313 | # Preprocess + encode |
| 314 | with torch.no_grad(): |
| 315 | if mode == "image": |
| 316 | if isinstance(input_data, str): |
| 317 | image = Image.open(input_data).convert("RGB") |
| 318 | else: |
| 319 | image = input_data.convert("RGB") |
| 320 | inputs = processor(images=image, return_tensors="pt").to(device) |
| 321 | features = model.get_image_features(**inputs) |
| 322 | |
| 323 | else: # text mode |
| 324 | # CLIP expects a list of strings |
| 325 | texts = [input_data] if isinstance(input_data, str) else list(input_data) |
| 326 | inputs = processor( |
| 327 | text=texts, |
| 328 | return_tensors="pt", |
| 329 | padding=True, |
| 330 | truncation=True, |
| 331 | max_length=processor.tokenizer.model_max_length, |
no test coverage detected