Function compute_motion_generalizability

mbench/motion_generalizability.py:25–141

Compute action accuracy for motion videos using Gemini API Args: full_info_path: Path to JSON file with video information device: Device to use (not used in this implementation) api_key: Gemini API key (can also be set via GEMINI_API_KEY env var) vlm_fps

(full_info_path: str, device: str,
                          api_key: Optional[str] = None, **kwargs)

def compute_motion_generalizability(full_info_path: str, device: str,
                          api_key: Optional[str] = None, **kwargs) -> Dict[str, Any]:
    """
    Compute action accuracy for motion videos using Gemini API
    
    Args:
        full_info_path: Path to JSON file with video information
        device: Device to use (not used in this implementation)
        api_key: Gemini API key (can also be set via GEMINI_API_KEY env var)
        vlm_fps: Target effective fps for VLM sampling (default: 5)
        **kwargs: Additional arguments
        
    Returns:
        Dictionary with evaluation results
    """
    # Get API key from parameter or environment
    if api_key is None:
        api_key = os.getenv('GEMINI_API_KEY')
    
    if not api_key:
        raise ValueError("Gemini API key must be provided either as parameter or GEMINI_API_KEY environment variable")
    
    # Load dimension information using the same pattern as motion_quality.py
    prompt_dict_ls = load_dimension_info(full_info_path, dimension='Motion_Generalizability')
    
    # Base prompt for motion analysis
    prompt = """
    You are given a rendered video showing captured human motion, with no background or interacting objects. First, analyze the human motion in the video in detail by describing the person's body posture, limb movements, and any repetitive or distinctive motion patterns you observe. 
    Then, compare the observed motion with the provided textual description of an expected motion. When evaluating, prioritize the semantic consistency of the action (e.g., whether the core intent and type of movement match) over the visual quality, smoothness, or physical realism of the rendering. Finally, determine whether the observed motion matches the described motion. Respond with a short justification followed by a final answer in "Answer: yes" or "Answer: no".
    Example output is: "The person in the video is standing and repeatedly clapping their hands above chest level. The motion in the video matches the given motion description 'clapping'. Answer: yes."
    The provided motion description to verify: """
    
    gemini_workers = kwargs.get('gemini_workers', 8)
    vlm_fps = kwargs.get('vlm_fps', 5)  # Default 1fps for VLM sampling
    
    cnt = 0
    cor_num = 0
    accuracy_list = []
    per_motion_metrics = []

    def evaluate_entry(prompt_dict):
        video_label = prompt_dict.get('prompt', 'unknown')
        video_path = prompt_dict.get('evaluation_file', '')
        motion_id = prompt_dict.get('id')

        if not os.path.exists(video_path):
            return {"skip": True, "id": motion_id, "error": f"Missing video {video_path}"}

        try:
            # Read video bytes directly - fps is now handled by Gemini API
            with open(video_path, 'rb') as f:
                video_bytes = f.read()
        except Exception as exc:
            return {"skip": True, "id": motion_id, "error": f"Video read error: {exc}"}


        try:
            response = call_gemini_api(video_bytes, prompt + f'"{video_label}".', api_key, fps=vlm_fps)