Method evaluate

spectrumlab/evaluator/open_evaluator.py:128–178

两阶段评测：先用被测模型生成答案，再用评分模型打分。 支持 score_model 参数。

(
        self,
        data_items,
        model,
        max_out_len=512,
        batch_size=None,
        save_path="./eval_results",
        score_model=None,
    )

        return prediction >= 0.5

    def evaluate(
        self,
        data_items,
        model,
        max_out_len=512,
        batch_size=None,
        save_path="./eval_results",
        score_model=None,
    ):
        """
        两阶段评测：先用被测模型生成答案，再用评分模型打分。
        支持 score_model 参数。
        """
        score_model = score_model or self.score_model
        results = []
        print("🚀 Running model inference...")
        model_outputs = []
        # 1. 让被测模型生成答案（带进度条）
        for item in tqdm(data_items, desc="Generating responses", unit="item"):
            prompt = self._build_prompt(item)
            model_output = model.generate(prompt, max_out_len)
            model_outputs.append(model_output)
        # 2. 评分阶段（带进度条）
        print("📝 Running scoring model...")
        for item, model_output in tqdm(
            zip(data_items, model_outputs),
            total=len(data_items),
            desc="Scoring responses",
            unit="item",
        ):
            score_prompt = self._build_score_prompt(item, model_output)
            score_response = score_model.generate(score_prompt, max_out_len)
            score = self._extract_prediction(score_response, item)
            # 3. 保存所有信息
            item_result = item.copy()
            item_result[self.prediction_key] = score
            item_result["model_output"] = model_output
            item_result["score_response"] = score_response
            item_result["pass"] = self._calculate_accuracy(
                item.get("answer", ""), score, item
            )
            results.append(item_result)
        # 4. 保存和统计
        saved_files = self._save_results(results, save_path)
        metrics = self._calculate_metrics(results)
        self._print_results(metrics)
        return {
            "metrics": metrics,
            "saved_files": saved_files,
            "total_items": len(results),
        }