Function process_data

eval/run_predict_aguvis.py:452–504

Process the data to run predict. Args: episode (Dict): one single data. image_path (str): the image_path processed. args(argparse.Namespace): the args passed in from terminal. Returns: Dict: the param used to run predict function.

(episode:Dict,
                image_path: str,
                args:argparse.Namespace)

def process_data(episode:Dict,
                image_path: str,
                args:argparse.Namespace) -> Dict:
    """
    Process the data to run predict.

    Args:
        episode (Dict): one single data.
        image_path (str): the image_path processed.
        args(argparse.Namespace): the args passed in from terminal.

    Returns:
        Dict: the param used to run predict function.
    """

    def load_image(image_file: str) -> Image:
        """
        Origin code for loading the image.

        Args:
            image_file (str): the path or the url of the image.

            Note we didn't do any resize action here.

        Returns:
            Image: the Image class.
        """
        if image_file.startswith(("http://", "https://")):
            response = requests.get(image_file)
            image = Image.open(BytesIO(response.content)).convert("RGB")
        else:
            image = Image.open(image_file).convert("RGB")
        return image

    image: Image = load_image(image_path)
    instruction:str = episode['instruction']
    low_instruction: Optional[str] = None

    data_name:str = args.data_name

    if data_name == 'android_control_low_test':
        low_instruction:str = episode['low_instruction']

    return {
        "image": image,
        "episode": episode,
        "instruction": instruction,
        "previous_actions": None,
        "low_level_instruction": low_instruction,
        "mode": args.mode,
        "temperature": args.temperature,
        "max_new_tokens": args.max_new_tokens
    }


def generate_response(