Loads model and processors (InstructBLIPProcessor contains Vicuna Tokenizer, Q-Former Tokenizer, and an ImageProcessor) using the HF `InstructBLIP*.from_pretrained()` functionality.
(self)
| 74 | self.string2indices[trigger_string] = token_idx_list |
| 75 | |
| 76 | def load(self) -> Tuple[nn.Module, Tokenizer, ImageProcessor]: |
| 77 | """ |
| 78 | Loads model and processors (InstructBLIPProcessor contains Vicuna Tokenizer, Q-Former Tokenizer, and an |
| 79 | ImageProcessor) using the HF `InstructBLIP*.from_pretrained()` functionality. |
| 80 | """ |
| 81 | with self.distributed_state.main_process_first(): |
| 82 | text_img_processor = InstructBlipProcessor.from_pretrained(self.hub_path) |
| 83 | model = InstructBlipForConditionalGeneration.from_pretrained(self.hub_path) |
| 84 | |
| 85 | # Lift `image_processor` for use in evaluation harnesses |
| 86 | image_processor = text_img_processor.image_processor |
| 87 | |
| 88 | # Place Model on Device |
| 89 | model = model.to(self.distributed_state.device, dtype=self.dtype) |
| 90 | model.eval() |
| 91 | |
| 92 | return model, text_img_processor, image_processor |
| 93 | |
| 94 | def freeze(self) -> None: |
| 95 | self.model.eval() |