Function evaluate_entry

mbench/motion_condition_consistency.py:65–110

(prompt_dict)

    per_motion_metrics = []

    def evaluate_entry(prompt_dict):
        video_label = prompt_dict.get('prompt', 'unknown')
        auxiliary_info = prompt_dict.get('auxiliary_info', [])
        video_path = prompt_dict.get('evaluation_file', '')
        motion_id = prompt_dict.get('id')

        if not os.path.exists(video_path):
            return {"skip": True, "id": motion_id, "error": f"Missing video {video_path}"}

        try:
            # Read video bytes directly - fps is now handled by Gemini API
            with open(video_path, 'rb') as f:
                video_bytes = f.read()
        except Exception as exc:
            return {"skip": True, "id": motion_id, "error": f"Video read error: {exc}"}


        motion_list = auxiliary_info.copy()
        if video_label not in motion_list:
            motion_list.append(video_label)
        random.shuffle(motion_list)
        result_string = ", ".join(motion_list)

        try:
            response = call_gemini_api(video_bytes, prompt + f'"{result_string}"', api_key, fps=vlm_fps)

        except Exception as exc:
            return {"skip": True, "id": motion_id, "error": f"Gemini error: {exc}"}

        response = response.encode('utf-8').decode('utf-8').strip()
        response_clean = response.split("\n\n", 1)[-1]
        response_clean = response_clean.split(":", 1)[-1].strip()
        # Remove trailing punctuation (., ", ', etc.) that may cause false negatives
        response_clean = response_clean.rstrip('.,;:!?\'"')

        is_correct = response_clean.lower() == video_label.lower()
        return {
            "skip": False,
            "id": motion_id,
            "is_correct": is_correct,
            "video_label": video_label,
            "video_path": video_path,
            "motion_list": motion_list,
            "response_clean": response_clean,
            "raw_response": response,
        }

    with ThreadPoolExecutor(max_workers=gemini_workers) as executor:
        futures = {executor.submit(evaluate_entry, prompt_dict): prompt_dict for prompt_dict in prompt_dict_ls}