Function INO_VLM_evaluation

evaluation/evaluate_INO_VLM.py:25–292

Args: region_target_height (int): Height without the padding, the padding will be scaled together region_target_width (int): Width without the padding, the padding will be scaled together test_num_frames (int): Sampled number of frames f

(data_parent_path, region_target_height, region_target_width, is_frame_in=True)

def INO_VLM_evaluation(data_parent_path, region_target_height, region_target_width, is_frame_in=True):
    '''
        Args:
            region_target_height (int):     Height without the padding, the padding will be scaled together
            region_target_width (int):      Width without the padding, the padding will be scaled together
            test_num_frames (int):          Sampled number of frames from the GT and GEN generated results
            is_frame_in (bool):             Whether frame_in or frame out
    '''


    # Prepare the pretrained weight
    vlm_model_path = "Qwen/Qwen2.5-VL-32B-Instruct"


    # Prepare the VLM Setting
    device = "cuda"
    llm_fps = 1 
    test_num_frames = 14            # Override it, Qwen cannot take in too many frames; 14 is a moderate value


    # Define the Instruction Prompt
    if not is_frame_in:     # Frame Out Instruction Prompt
        instruction_prompt = "Please check if the object leave the frame. Return a Yes/No as the only response."
    else:   # Frame In
        instruction_prompt = "Please check if the object enter the frame. Return a Yes/No as the only response."


    # Messages containing a local video path and a text query
    messages = [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "video",
                                # "video": video_path,
                                "max_pixels": 360 * 420,            # The video information here should be deprecated
                                "fps": llm_fps,
                            },
                            {
                                "type": "text", 
                                "text": instruction_prompt,
                            },
                        ],
                    }
                ]



    # Init the QWen VLM Model
    processor = AutoProcessor.from_pretrained(vlm_model_path)
    bnb_config = BitsAndBytesConfig(
                                        load_in_4bit=True,
                                        bnb_4bit_compute_dtype=torch.float16,  # Use float16 for computations
                                        bnb_4bit_use_double_quant=True,
                                        bnb_4bit_quant_type='nf4',  # NormalFloat4 quantization
                                    )
    model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
                                                                vlm_model_path,