Method evaluate_many

spectrumlab/evaluator/base.py:99–249

Evaluate a single model on data_items with parallel processing. Args: data_items: List of data items to evaluate model: Model instance to evaluate max_out_len: Maximum output length for model generation batch_size: Batch size for proc

(
        self,
        data_items: List[Dict],
        model,
        max_out_len: int = 512,
        batch_size: Optional[int] = None,
        save_path: str = "./eval_results",
        n_jobs: int = -1,
    )

        }

    def evaluate_many(
        self,
        data_items: List[Dict],
        model,
        max_out_len: int = 512,
        batch_size: Optional[int] = None,
        save_path: str = "./eval_results",
        n_jobs: int = -1,
    ) -> Dict:
        """
        Evaluate a single model on data_items with parallel processing.

        Args:
            data_items: List of data items to evaluate
            model: Model instance to evaluate
            max_out_len: Maximum output length for model generation
            batch_size: Batch size for processing (if None, will be auto-calculated)
            save_path: Base path to save results
            n_jobs: Number of parallel jobs (-1 for all available cores)

        Returns:
            Dictionary containing evaluation results
        """
        import multiprocessing as mp
        from concurrent.futures import ThreadPoolExecutor, as_completed
        import math

        if not data_items:
            print("❌ No data items provided")
            return {"error": "No data items provided"}

        # Set number of jobs
        if n_jobs == -1:
            n_jobs = mp.cpu_count()

        # Calculate batch size if not provided
        if batch_size is None:
            batch_size = max(1, math.ceil(len(data_items) / n_jobs))

        print(f"🔄 Starting parallel evaluation on {len(data_items)} items...")
        print(f"📝 Model: {type(model).__name__}")
        print(f"⚡ Using {n_jobs} parallel workers with batch size {batch_size}")

        # Split data into batches
        batches = [
            data_items[i : i + batch_size]
            for i in range(0, len(data_items), batch_size)
        ]

        print(f"📦 Split into {len(batches)} batches")

        # Build prompts for all items
        print("📝 Building prompts...")
        all_prompts = [self._build_prompt(item) for item in data_items]

        # Split prompts into batches
        prompt_batches = [
            all_prompts[i : i + batch_size]