Method evaluate

spectrumlab/evaluator/base.py:25–97

(
        self,
        data_items: List[Dict],
        model,
        max_out_len: int = 512,
        batch_size: Optional[int] = None,
        save_path: str = "./eval_results",
    )

        pass

    def evaluate(
        self,
        data_items: List[Dict],
        model,
        max_out_len: int = 512,
        batch_size: Optional[int] = None,
        save_path: str = "./eval_results",
    ) -> Dict:
        if not data_items:
            print("❌ No data items provided")
            return {"error": "No data items provided"}

        print(f"🔄 Starting evaluation on {len(data_items)} items...")
        print(f"📝 Model: {type(model).__name__}")

        # 1. Build prompts
        print("📝 Building prompts...")
        prompts = [self._build_prompt(item) for item in data_items]

        # 2. Run model inference
        print("🚀 Running model inference...")
        responses = []
        try:
            # 统一使用sequential generation with progress bar
            for i, prompt in enumerate(
                tqdm(prompts, desc="Generating responses", unit="item")
            ):
                try:
                    response = model.generate(prompt, max_out_len)
                    responses.append(response)
                except Exception as e:
                    print(f"\n⚠️  Error on item {i + 1}: {e}")
                    responses.append(f"Error: {str(e)}")

        except Exception as e:
            return {"error": f"Model generation failed: {e}"}

        # 3. Extract predictions and add to data
        print("🔍 Extracting predictions...")
        processed_items = []
        for item, response in tqdm(
            zip(data_items, responses),
            desc="Processing responses",
            total=len(data_items),
            unit="item",
        ):
            item_copy = item.copy()
            prediction = self._extract_prediction(response, item)
            item_copy[self.prediction_key] = prediction
            item_copy["model_response"] = response

            answer = item.get("answer", "")
            is_correct = self._calculate_accuracy(answer, prediction, item)
            item_copy["pass"] = is_correct

            processed_items.append(item_copy)

        # 4. Save results