| 23 | |
| 24 | |
| 25 | class InstructBLIP(VLM): |
| 26 | def __init__( |
| 27 | self, |
| 28 | hub_path: Path, |
| 29 | max_length: int = 512, |
| 30 | temperature: float = 0.2, |
| 31 | **_: str, |
| 32 | ) -> None: |
| 33 | self.hub_path = hub_path |
| 34 | self.dtype = torch.float32 |
| 35 | |
| 36 | # Get Distributed State |
| 37 | self.distributed_state = PartialState() |
| 38 | |
| 39 | # Load Model on GPU(s) --> download if necessary via HF Hub |
| 40 | self.model, self.text_img_processor, self.image_processor = self.load() |
| 41 | resize = (self.image_processor.size["height"], self.image_processor.size["width"]) |
| 42 | self.image_processor_from_tensor = Compose( |
| 43 | [ |
| 44 | Resize(resize, interpolation=InterpolationMode.BICUBIC, antialias=True), |
| 45 | Normalize(mean=self.image_processor.image_mean, std=self.image_processor.image_std), |
| 46 | ] |
| 47 | ) |
| 48 | |
| 49 | # For Fair Evaluation against LLaVa/Quartz/IDEFICS --> greedy decoding: |
| 50 | self.max_length = max_length |
| 51 | self.temperature = temperature |
| 52 | self.generate_kwargs = {"do_sample": False, "max_new_tokens": self.max_length, "temperature": self.temperature} |
| 53 | |
| 54 | # InstructBLIP Default Generation Configuration =>> Uses Beam Search (very slow!) |
| 55 | # => Ref: https://huggingface.co/Salesforce/instructblip-vicuna-7b#intended-uses--limitations |
| 56 | # self.generate_kwargs = { |
| 57 | # "do_sample": False, |
| 58 | # "num_beams": 5, |
| 59 | # "max_length": self.max_length, |
| 60 | # "min_length": 1, |
| 61 | # "repetition_penalty": 1.5, |
| 62 | # "length_penalty": 1.0, |
| 63 | # "temperature": 1, |
| 64 | # } |
| 65 | |
| 66 | # For computing likelihoods --> get tokens corresponding to "true", "false" and "yes", "no" |
| 67 | # self.string2idx = {} |
| 68 | self.string2indices = {} |
| 69 | for trigger_string in ["true", "false", "yes", "no"] + [chr(ord("A") + i) for i in range(26)]: |
| 70 | token_idx_list = self.text_img_processor.tokenizer.encode(trigger_string, add_special_tokens=False) |
| 71 | print(f"Trigger: {trigger_string} --> {token_idx_list}") |
| 72 | # assert len(token_idx_list) == 1, f'String "{trigger_string}" is tokenized as more than one token!' |
| 73 | # self.string2idx[trigger_string] = token_idx_list[0] |
| 74 | self.string2indices[trigger_string] = token_idx_list |
| 75 | |
| 76 | def load(self) -> Tuple[nn.Module, Tokenizer, ImageProcessor]: |
| 77 | """ |
| 78 | Loads model and processors (InstructBLIPProcessor contains Vicuna Tokenizer, Q-Former Tokenizer, and an |
| 79 | ImageProcessor) using the HF `InstructBLIP*.from_pretrained()` functionality. |
| 80 | """ |
| 81 | with self.distributed_state.main_process_first(): |
| 82 | text_img_processor = InstructBlipProcessor.from_pretrained(self.hub_path) |