Function mapping_actions

eval/run_predict_aguvis.py:218–399

Mapping the string from aguvis model into minicpm Action space. Args: episode (dict): The episode dict, containing all information and a prediction string. Returns: the episode whose prediction string is mapped into minicpm Action space. In practice the model will

(episode: dict)

# action mapping begins. ===========
def mapping_actions(episode: dict) -> dict:
    """
    Mapping the string from aguvis model into minicpm Action space.

    Args:
        episode (dict): The episode dict, containing all information and a prediction string.

    Returns:
        the episode whose prediction string is mapped into minicpm Action space.

    In practice the model will output unstable strings. We only handle those stable cases.
    """

    FAIL_PARSE = {
        "STATUS": "FAIL"
    }

    action:str = episode["pred"].split('\n')[-1].strip()

    platform = action.split('.')[0]

    function = action[len(platform) + 1 :]

    if platform == "pyautogui":

        if function.startswith("click"):
            # deal with click function.
            try:
                matches = re.findall(r"[-+]?\d*\.\d+|\d+", function)
                x,y = matches
                x = round(float(x) * 1000)
                y = round(float(y) * 1000)

                episode["pred"] = {
                    "POINT": [x, y],
                    "duration": 200,
                    "STATUS": "continue"
                }
            except Exception as e:
                print(f"Failed to parse POINT ACTION {function}: {e}")
                episode["pred"] = FAIL_PARSE

        elif function.startswith("write"):
            # deal with type action.
            try:

                pattern = r'message=(["\'])(.*?)\1'
                match = re.search(pattern, function)

                text = match.group(2)

                episode["pred"] = {
                    "TYPE": text,
                    "duration": 200,
                    "STATUS": "continue"
                }

            except Exception as e: