Modified from the origin code. Do the inference based on the params. Args: image (Image.Image) instruction (str) previous_actions (Optional[str | List[str]], optional) low_level_instruction (Optional[str], optional) mode (Literal[], optional)
(
image: Image.Image,
episode: Dict,
instruction: str,
previous_actions: Optional[str | List[str]] = None,
low_level_instruction: Optional[str] = None,
mode: Literal["self-plan", "force-plan", "grounding"] = "self-plan",
temperature: float = 0,
max_new_tokens: int = 1024)
| 505 | |
| 506 | |
| 507 | def generate_response( |
| 508 | image: Image.Image, |
| 509 | episode: Dict, |
| 510 | instruction: str, |
| 511 | previous_actions: Optional[str | List[str]] = None, |
| 512 | low_level_instruction: Optional[str] = None, |
| 513 | mode: Literal["self-plan", "force-plan", "grounding"] = "self-plan", |
| 514 | temperature: float = 0, |
| 515 | max_new_tokens: int = 1024) -> str: |
| 516 | """ |
| 517 | Modified from the origin code. Do the inference based on the params. |
| 518 | |
| 519 | Args: |
| 520 | image (Image.Image) |
| 521 | instruction (str) |
| 522 | previous_actions (Optional[str | List[str]], optional) |
| 523 | low_level_instruction (Optional[str], optional) |
| 524 | mode (Literal[], optional) |
| 525 | temperature (float, optional) |
| 526 | max_new_tokens (int, optional) |
| 527 | |
| 528 | Returns: |
| 529 | str: the predict result. |
| 530 | """ |
| 531 | |
| 532 | global _llm, _processor, _tokenizer |
| 533 | |
| 534 | # process the prompt. |
| 535 | system_message = { |
| 536 | "role": "system", |
| 537 | "content": grounding_system_message if mode == "grounding" else agent_system_message, |
| 538 | } |
| 539 | |
| 540 | if isinstance(previous_actions, list): |
| 541 | previous_actions = "\n".join(previous_actions) |
| 542 | if not previous_actions: |
| 543 | previous_actions = "None" |
| 544 | |
| 545 | user_message = { |
| 546 | "role": "user", |
| 547 | "content": [ |
| 548 | {"type": "image", "image": image}, |
| 549 | { |
| 550 | "type": "text", |
| 551 | "text": user_instruction.format( |
| 552 | overall_goal=instruction, |
| 553 | previous_actions=previous_actions, |
| 554 | low_level_instruction=low_level_instruction, |
| 555 | ), |
| 556 | }, |
| 557 | ], |
| 558 | } |
| 559 | |
| 560 | if low_level_instruction: |
| 561 | recipient_text = f"<|im_start|>assistant<|recipient|>all\nAction: {low_level_instruction}\n" |
| 562 | elif mode == "grounding": |
| 563 | recipient_text = "<|im_start|>assistant<|recipient|>os\n" |
| 564 | elif mode == "self-plan": |