Function evaluate_dataset

tasks/AutoMem/run_0/code/eval.py:352–523

Evaluate the agent on the LoComo dataset with parallel processing. Args: dataset_path: Path to the dataset file model: Name of the model to use output_path: Path to save results ratio: Ratio of dataset to evaluate backend: Backend to use (openai, ollama,

(dataset_path: str, model: str, output_path: Optional[str] = None,
                    ratio: float = 1.0, backend: str = "sglang", temperature_c5: float = 0.5,
                    retrieve_k: int = 10, sglang_host: str = "http://localhost",
                    sglang_port: int = 30000, num_workers: int = 4, out_dir: Optional[str] = None)

    }

def evaluate_dataset(dataset_path: str, model: str, output_path: Optional[str] = None,
                    ratio: float = 1.0, backend: str = "sglang", temperature_c5: float = 0.5,
                    retrieve_k: int = 10, sglang_host: str = "http://localhost",
                    sglang_port: int = 30000, num_workers: int = 4, out_dir: Optional[str] = None) -> dict:
    """Evaluate the agent on the LoComo dataset with parallel processing.

    Args:
        dataset_path: Path to the dataset file
        model: Name of the model to use
        output_path: Path to save results
        ratio: Ratio of dataset to evaluate
        backend: Backend to use (openai, ollama, sglang)
        temperature_c5: Temperature for category 5 questions
        retrieve_k: Number of memories to retrieve
        sglang_host: SGLang server host
        sglang_port: SGLang server port
        num_workers: Number of parallel workers
    """
    # Generate automatic log filename with timestamp
    timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M")
    log_filename = f"eval_ours_{model}_{backend}_ratio{ratio}_parallel_{timestamp}.log"
    log_path = os.path.join(os.path.dirname(__file__), "logs", log_filename)

    # Create logs directory if it doesn't exist
    os.makedirs(os.path.dirname(log_path), exist_ok=True)

    logger = setup_logger(log_path)
    logger.info(f"Loading dataset from {dataset_path}")

    # Load dataset
    samples = load_locomo_dataset(dataset_path)
    logger.info(f"Loaded {len(samples)} samples")

    # Select subset of samples based on ratio
    if ratio < 1.0:
        num_samples = max(1, int(len(samples) * ratio))
        samples = samples[:num_samples]
        logger.info(f"Using {num_samples} samples ({ratio*100:.1f}% of dataset)")

    # Create memories directory
    memories_dir = os.path.join(
        os.path.dirname(__file__),
        f"cached_memories_advanced_{backend}_{model}"
    )
    os.makedirs(memories_dir, exist_ok=True)

    allow_categories = [1, 2, 3, 4, 5]

    # Prepare sample data for parallel processing
    sample_data_list = [(idx, sample) for idx, sample in enumerate(samples)]

    logger.info(f"Starting parallel processing with {num_workers} workers")

    # Use partial to fix the constant arguments
    process_func = partial(
        process_single_sample,
        model=model,
        backend=backend,