(self,input_imgs,detect_model)
| 442 | |
| 443 | @torch.no_grad() |
| 444 | def ask_question(self,input_imgs,detect_model): |
| 445 | left_view = input_imgs[0] |
| 446 | right_view = input_imgs[1] |
| 447 | top_view = input_imgs[2] |
| 448 | # print("#############") |
| 449 | vision_feature = self.get_image_embedding(left_view) |
| 450 | left_glip = np.array(left_view) |
| 451 | right_glip = np.array(right_view) |
| 452 | top_glip = np.array(top_view) |
| 453 | self.conv = get_conv_template("husky").copy() |
| 454 | modal_type = "image" |
| 455 | conversations = self.ask(text="Please describe this image in detail and focus on the parts are interactive.", conv=self.conv, modal_type=modal_type) |
| 456 | caption = self.answer(conversations, vision_feature, modal_type=modal_type) |
| 457 | print("caption:",caption) |
| 458 | self.conv.messages[-1][1] = caption.strip() |
| 459 | conversations = self.ask(text="List all visible objects or object-parts in a single line with brief labels only and separate them by commas. Pithy!!!", conv=self.conv, modal_type=modal_type) |
| 460 | object_list = self.answer(conversations, vision_feature, modal_type=modal_type) |
| 461 | object_list = "red apple, green pear" |
| 462 | print("object_list",object_list) |
| 463 | left_dict = self.glip_show(detect_model,left_glip, object_list,view="left") |
| 464 | right_dict = self.glip_show(detect_model,right_glip, object_list,view="right") |
| 465 | top_dict = self.glip_show(detect_model,top_glip, object_list,view="top") |
| 466 | # final_dict = self.merge_box(left_dict,right_dict,top_dict) |
| 467 | # return final_dict |
| 468 | |
| 469 | |
| 470 |
no test coverage detected