(self, original_user_request, step_num)
| 14 | return json_instructions |
| 15 | |
| 16 | def format_user_request_for_llm(self, original_user_request, step_num) -> list[dict[str, Any]]: |
| 17 | base64_img: str = Screen().get_screenshot_in_base64() |
| 18 | |
| 19 | request_data: str = json.dumps({ |
| 20 | 'original_user_request': original_user_request, |
| 21 | 'step_num': step_num |
| 22 | }) |
| 23 | |
| 24 | # We have to add context every request for now which is expensive because our chosen model doesn't have a |
| 25 | # stateful/Assistant mode yet. |
| 26 | message = [ |
| 27 | {'type': 'text', 'text': self.context + request_data}, |
| 28 | {'type': 'image_url', |
| 29 | 'image_url': { |
| 30 | 'url': f'data:image/jpeg;base64,{base64_img}' |
| 31 | } |
| 32 | } |
| 33 | ] |
| 34 | |
| 35 | return message |
| 36 | |
| 37 | def send_message_to_llm(self, message) -> ChatCompletion: |
| 38 | response = self.client.chat.completions.create( |
no test coverage detected