Function process_step_data

eval/run_eval_agent.py:94–138

Process a single step of data, load prediction results and evaluate. Args: step_data (dict): Data of a single step. Returns: dict or None: None means the step hasn't been predicted yet. Empty JSON indicates JSON parsing failed.

(step_data, evaluator, save_dir)

        return self.data[index]

def process_step_data(step_data, evaluator, save_dir):
    """
    Process a single step of data, load prediction results and evaluate.

    Args:
        step_data (dict): Data of a single step.

    Returns:
        dict or None: None means the step hasn't been predicted yet. Empty JSON indicates JSON parsing failed.
    """

    subset = step_data.get('subset')
    episode_id = step_data.get('episode_id')
    step_id = step_data.get('step_id')

    if subset is None or episode_id is None or step_id is None:
        raise ValueError(f"Missing subset/episode_id/step_id in test step data: {step_data}")

    save_dir_ep = os.path.join(save_dir, f"{subset}-{episode_id}")
    cur_save_path = os.path.join(save_dir_ep, f"{subset}-{episode_id}_{step_id}.json")

    try:
        # Ensure directory exists
        os.makedirs(os.path.dirname(cur_save_path), exist_ok=True)

        # Check if the file exists; if not, the step hasn't been predicted yet
        if not os.path.exists(cur_save_path):
            return None

        # Load the existing file
        with open(cur_save_path, "r") as file:
            try:
                pred = json.load(file)
            except json.JSONDecodeError as e:
                logging.error(f"JSON decoding failed, file: {cur_save_path}, error: {e}")
                pred = {'action_predict': {'COA': {'txt': {'ACTION': None, 'ARGS': None, 'STATUS': None}}}}

        assert pred is not None

        # Use global evaluator to evaluate
        result = evaluator(step_data, pred)
        return result

    except Exception as e:
        raise  ExecError(f"An error occurred, indicating unhandled edge case.")

def evaluate(args):