Data model for benchmark statistics
| 45 | extra: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional metadata or intermediate evaluation metrics") |
| 46 | |
| 47 | class Stats(BaseModel): |
| 48 | """Data model for benchmark statistics""" |
| 49 | model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") |
| 50 | |
| 51 | accuracy: float = Field(default=0.0, description="Overall accuracy score") |
| 52 | total: int = Field(default=0, description="Total number of tasks") |
| 53 | correct: int = Field(default=0, description="Number of correct tasks") |
| 54 | wrong: int = Field(default=0, description="Number of wrong tasks") |
| 55 | times: Dict[str, float] = Field(default_factory=dict, description="Time taken for each task (task_id -> seconds)") |
| 56 | average_time: float = Field(default=0.0, description="Average time per task in seconds") |
| 57 | extra: Dict[str, Any] = Field(default_factory=dict, description="Additional statistics or information") |
| 58 | |
| 59 | class Benchmark(BaseModel): |
| 60 | """Base class for all benchmark systems""" |