(image_input, state, visual_chatgpt=None)
| 133 | |
| 134 | |
| 135 | def upload_callback(image_input, state, visual_chatgpt=None): |
| 136 | |
| 137 | if isinstance(image_input, dict): # if upload from sketcher_input, input contains image and mask |
| 138 | image_input, mask = image_input['image'], image_input['mask'] |
| 139 | |
| 140 | click_state = [[], [], []] |
| 141 | image_input = image_resize(image_input, res=1024) |
| 142 | |
| 143 | model = build_caption_anything_with_models( |
| 144 | args, |
| 145 | api_key="", |
| 146 | captioner=shared_captioner, |
| 147 | sam_model=shared_sam_model, |
| 148 | ocr_reader=shared_ocr_reader, |
| 149 | session_id=iface.app_id |
| 150 | ) |
| 151 | model.segmenter.set_image(image_input) |
| 152 | image_embedding = model.image_embedding |
| 153 | original_size = model.original_size |
| 154 | input_size = model.input_size |
| 155 | |
| 156 | if visual_chatgpt is not None: |
| 157 | print('upload_callback: add caption to chatGPT memory') |
| 158 | new_image_path = get_new_image_name('chat_image', func_name='upload') |
| 159 | image_input.save(new_image_path) |
| 160 | visual_chatgpt.current_image = new_image_path |
| 161 | img_caption = model.captioner.inference(image_input, filter=False, args={'text_prompt':''})['caption'] |
| 162 | Human_prompt = f'\nHuman: The description of the image with path {new_image_path} is: {img_caption}. This information helps you to understand this image, but you should use tools to finish following tasks, rather than directly imagine from my description. If you understand, say \"Received\". \n' |
| 163 | AI_prompt = "Received." |
| 164 | visual_chatgpt.global_prompt = Human_prompt + 'AI: ' + AI_prompt |
| 165 | visual_chatgpt.agent.memory.buffer = visual_chatgpt.agent.memory.buffer + visual_chatgpt.global_prompt |
| 166 | state = [(None, 'Received new image, resize it to width {} and height {}: '.format(image_input.size[0], image_input.size[1]))] |
| 167 | |
| 168 | return state, state, image_input, click_state, image_input, image_input, image_embedding, \ |
| 169 | original_size, input_size |
| 170 | |
| 171 | |
| 172 | def inference_click(image_input, point_prompt, click_mode, enable_wiki, language, sentiment, factuality, |
nothing calls this directly
no test coverage detected