| 303 | |
| 304 | ## MMLU ## |
| 305 | class CustomMMLUEvaluationTask(LightevalTaskConfig): |
| 306 | def __init__( |
| 307 | self, |
| 308 | name, |
| 309 | prompt_function=None, |
| 310 | hf_repo="lighteval/mmlu", |
| 311 | hf_subset=None, |
| 312 | # metric=[Metrics.loglikelihood_acc_single_token], |
| 313 | metric=[Metrics.loglikelihood_acc, Metrics.loglikelihood_acc_norm_nospace], |
| 314 | hf_avail_splits=None, |
| 315 | evaluation_splits=["test"], |
| 316 | few_shots_split="dev", |
| 317 | few_shots_select=None, |
| 318 | suite=["custom"], |
| 319 | generation_size=-1, |
| 320 | stop_sequence=None, |
| 321 | output_regex=None, |
| 322 | frozen=False, |
| 323 | ): |
| 324 | super().__init__( |
| 325 | name=name, |
| 326 | prompt_function=prompt_function, |
| 327 | suite=suite, |
| 328 | hf_repo=hf_repo, |
| 329 | hf_subset=hf_subset, |
| 330 | metric=metric, |
| 331 | hf_avail_splits=hf_avail_splits, |
| 332 | evaluation_splits=evaluation_splits, |
| 333 | few_shots_split=few_shots_split, |
| 334 | few_shots_select=few_shots_select, |
| 335 | generation_size=generation_size, |
| 336 | stop_sequence=stop_sequence, |
| 337 | output_regex=output_regex, |
| 338 | frozen=frozen, |
| 339 | ) |
| 340 | |
| 341 | |
| 342 | if __name__ == "__main__": |
nothing calls this directly
no outgoing calls
no test coverage detected