(self,detect_model,mod="detect")
| 470 | |
| 471 | @torch.no_grad() |
| 472 | def post_question(self,detect_model,mod="detect"): |
| 473 | image_state = False |
| 474 | video_state = False |
| 475 | print("#############") |
| 476 | print("please input the query!") |
| 477 | query = input("\n") |
| 478 | if query.lower().endswith(('.bmp', '.dib', '.png', '.jpg', '.jpeg', '.pbm', '.pgm', '.ppm', '.tif', '.tiff')): |
| 479 | if os.path.exists(query): |
| 480 | print("received.") |
| 481 | image=read_image(query) |
| 482 | vision_feature = self.get_image_embedding(image) |
| 483 | image_glip = np.array(image) |
| 484 | self.conv = get_conv_template("husky").copy() |
| 485 | image_state = True |
| 486 | if query.lower().endswith(('.mp4', '.mkv', '.avi', '.wmv', '.iso', ".webm")): |
| 487 | if os.path.exists(query): |
| 488 | print("received.") |
| 489 | vision_feature = self.get_video_embedding(query) |
| 490 | self.conv = get_conv_template("husky").copy() |
| 491 | video_state = True |
| 492 | if image_state: |
| 493 | modal_type = "image" |
| 494 | elif video_state: |
| 495 | modal_type = "video" |
| 496 | else: |
| 497 | modal_type = "text" |
| 498 | |
| 499 | # if mod=="caption": |
| 500 | # conversations = self.ask(text="please describe this image in detail", conv=self.conv, modal_type=modal_type) |
| 501 | # caption = self.answer(conversations, vision_feature, modal_type=modal_type) |
| 502 | # print("mod is caption")List all visible objects and visible object-parts in a single line only and separate them by semicolons. Pithy!!! |
| 503 | # print(caption) |
| 504 | |
| 505 | if mod=="detect": |
| 506 | conversations = self.ask(text="Please describe this image in detail and focus on the parts are interactive.", conv=self.conv, modal_type=modal_type) |
| 507 | caption = self.answer(conversations, vision_feature, modal_type=modal_type) |
| 508 | self.conv.messages[-1][1] = caption.strip() |
| 509 | conversations = self.ask(text="List all visible objects or object-parts in a single line with brief labels only and separate them by commas. Pithy!!!", conv=self.conv, modal_type=modal_type) |
| 510 | #object_list = self.answer(conversations, vision_feature, modal_type=modal_type) |
| 511 | |
| 512 | scores, boxes, names = detect_model.inference_on_image(image_glip, object_list) |
| 513 | scores, boxes, names = self.reduce_boxes(scores, boxes, names) |
| 514 | box_dict=self.refine_bbox_dict(boxes, names) |
| 515 | # draw output image |
| 516 | plt.figure(figsize=(10, 10)) |
| 517 | # image_rbg = cv2.cvtColor(image_glip, cv2.COLOR_BGR2RGB) |
| 518 | plt.imshow(image_glip) |
| 519 | show_predictions(scores, boxes, names) |
| 520 | plt.axis('off') |
| 521 | plt.savefig("./test_glip.png",bbox_inches="tight", dpi=300, pad_inches=0.0) |
| 522 | print("mod is detection and caption") |
| 523 | print(caption) |
| 524 | print(object_list) |
| 525 | print("box:",boxes) |
| 526 | print("names:",names) |
| 527 | |
| 528 | |
| 529 |
nothing calls this directly
no test coverage detected