| 776 | |
| 777 | |
| 778 | class VisualQuestionAnswering: |
| 779 | def __init__(self, device): |
| 780 | print(f"Initializing VisualQuestionAnswering to {device}") |
| 781 | self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32 |
| 782 | self.device = device |
| 783 | self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") |
| 784 | self.model = BlipForQuestionAnswering.from_pretrained( |
| 785 | "Salesforce/blip-vqa-base", torch_dtype=self.torch_dtype).to(self.device) |
| 786 | |
| 787 | @prompts(name="Answer Question About The Image", |
| 788 | description="useful when you need an answer for a question based on an image. " |
| 789 | "like: what is the background color of the last image, how many cats in this figure, what is in this figure. " |
| 790 | "The input to this tool should be a comma separated string of two, representing the image_path and the question") |
| 791 | def inference(self, inputs): |
| 792 | image_path, question = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) |
| 793 | raw_image = Image.open(image_path).convert('RGB') |
| 794 | inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device, self.torch_dtype) |
| 795 | out = self.model.generate(**inputs) |
| 796 | answer = self.processor.decode(out[0], skip_special_tokens=True) |
| 797 | print(f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, " |
| 798 | f"Output Answer: {answer}") |
| 799 | return answer |
| 800 | |
| 801 | |
| 802 | class Segmenting: |
nothing calls this directly
no outgoing calls
no test coverage detected