Evaluates world model benchmark videos using VILA model.
| 112 | |
| 113 | |
| 114 | class WorldModelEvaluator: |
| 115 | """Evaluates world model benchmark videos using VILA model.""" |
| 116 | |
| 117 | def __init__(self, judge_path: str, video_dir: str, config: EvaluationConfig): |
| 118 | self.judge = self._load_judge(judge_path) |
| 119 | self.video_dir = Path(video_dir) |
| 120 | self.config = config |
| 121 | self.logger = logging.getLogger(__name__) |
| 122 | self.printer = ResultsPrinter() |
| 123 | |
| 124 | @staticmethod |
| 125 | def _load_judge(judge_path: str): |
| 126 | """Load the VILA judge model.""" |
| 127 | import llava |
| 128 | return llava.load(judge_path) |
| 129 | |
| 130 | def _load_video(self, video_name: str) -> Optional['llava.Video']: |
| 131 | """Load a video file for evaluation.""" |
| 132 | video_path = self.video_dir / f"{video_name}.mp4" |
| 133 | if not video_path.exists(): |
| 134 | self.logger.warning(f"Video not found: {video_path}") |
| 135 | return None |
| 136 | import llava |
| 137 | return llava.Video(str(video_path)) |
| 138 | |
| 139 | def evaluate_video(self, video: 'llava.Video', prompt: str, cot: bool = True) -> str: |
| 140 | """Generate evaluation content for a video.""" |
| 141 | if not cot: |
| 142 | prompt = prompt.replace( |
| 143 | "Let's think step-by-step and conclude with", "Answer with" |
| 144 | ).replace( |
| 145 | "Let's analyze step-by-step and conclude with", "Answer with" |
| 146 | ) |
| 147 | return self.judge.generate_content([video, prompt]) |
| 148 | |
| 149 | def process_results(self, preds: Dict, accs: defaultdict) -> float: |
| 150 | """Process and print evaluation results with rich formatting.""" |
| 151 | num_insts = len(preds) |
| 152 | total_score = 0 |
| 153 | |
| 154 | category_mapping = { |
| 155 | 2: [("framewise", "temporal")], |
| 156 | 5: [("newton", "mass", "fluid", "penetration", "gravity")] |
| 157 | } |
| 158 | |
| 159 | for category, scores in accs.items(): |
| 160 | self.printer.print_header(f"{category.replace('_', ' ').title()} Details") |
| 161 | num_sub = len(scores) // num_insts |
| 162 | |
| 163 | if num_sub == 1: |
| 164 | overall = np.mean(scores) |
| 165 | self.printer.print_score("Overall", overall) |
| 166 | total_score += overall |
| 167 | elif num_sub in category_mapping: |
| 168 | sub_scores = {} |
| 169 | for i, sub in enumerate(category_mapping[num_sub][0]): |
| 170 | sub_mean = np.mean(scores[i::num_sub]) |
| 171 | sub_scores[sub.title()] = sub_mean |