(content, screenshot_file)
| 206 | return |
| 207 | |
| 208 | def select(content, screenshot_file): |
| 209 | prompt_select = get_select_prompt(content) |
| 210 | chat_select = init_action_chat() |
| 211 | chat_select = add_response("user", prompt_select, chat_select, [screenshot_file]) |
| 212 | output_select = inference_chat(chat_select, vl_model_version, API_url, token) |
| 213 | print(output_select) |
| 214 | first_line = output_select.split('<first>')[-1].split('</first>')[0][:30] |
| 215 | last_line = output_select.split('<last>')[-1].split('</last>')[0][-30:] |
| 216 | time.sleep(2) |
| 217 | |
| 218 | text_sys = OpenOCR(mode='mobile', drop_score=0.5, |
| 219 | det_box_type='quad') # det_box_type: 'quad' or 'poly' |
| 220 | res, _ = text_sys(img_path=screenshot_file, save_dir='e2e_results/', is_visualize=False) |
| 221 | |
| 222 | for item in res[0]: |
| 223 | if first_line in item['transcription']: |
| 224 | corr_first = item['points'][0] |
| 225 | break |
| 226 | for item in res[0]: |
| 227 | if last_line in item['transcription']: |
| 228 | corr_last = item['points'][2] |
| 229 | break |
| 230 | |
| 231 | x1, y1 = corr_first |
| 232 | x2, y2 = corr_last |
| 233 | drag(x1, y1, x2, y2) |
| 234 | return |
| 235 | |
| 236 | |
| 237 | def drag(x1, y1, x2, y2): |
no test coverage detected