| 43 | |
| 44 | @dataclass |
| 45 | class TaskConfig(dict): |
| 46 | # task naming/registry |
| 47 | task: str = None |
| 48 | task_alias: str = None |
| 49 | group: Union[str, list] = None |
| 50 | group_alias: Union[str, list] = None |
| 51 | # HF dataset options. |
| 52 | # which dataset to use, |
| 53 | # and what splits for what purpose |
| 54 | dataset_path: str = None |
| 55 | dataset_name: str = None |
| 56 | dataset_kwargs: dict = None |
| 57 | training_split: str = None |
| 58 | validation_split: str = None |
| 59 | test_split: str = None |
| 60 | fewshot_split: str = None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaling (?) |
| 61 | # formatting / prompting options. |
| 62 | # see docs/advanced_task_guide.md for more info |
| 63 | process_docs: Callable = None |
| 64 | doc_to_text: Union[Callable, str] = None |
| 65 | doc_to_target: Union[Callable, str] = None |
| 66 | doc_to_choice: Union[Callable, str, dict, list] = None |
| 67 | process_results: Union[Callable, str] = None |
| 68 | use_prompt: str = None |
| 69 | description: str = "" |
| 70 | target_delimiter: str = " " |
| 71 | fewshot_delimiter: str = "\n\n" |
| 72 | fewshot_config: dict = None |
| 73 | # runtime configuration options |
| 74 | num_fewshot: int = None |
| 75 | # scoring options |
| 76 | metric_list: list = None |
| 77 | output_type: Literal[ |
| 78 | "loglikelihood", |
| 79 | "loglikelihood_rolling", |
| 80 | "generate_until", |
| 81 | "multiple_choice", |
| 82 | ] = "generate_until" |
| 83 | generation_kwargs: dict = None |
| 84 | repeats: int = 1 |
| 85 | filter_list: Union[str, list] = None |
| 86 | should_decontaminate: bool = False |
| 87 | doc_to_decontamination_query: str = None |
| 88 | metadata: dict = None # by default, not used in the code. allows for users to pass arbitrary info to tasks |
| 89 | |
| 90 | def __post_init__(self) -> None: |
| 91 | if self.generation_kwargs is not None: |
| 92 | if self.output_type != "generate_until": |
| 93 | eval_logger.warning( |
| 94 | f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!" |
| 95 | ) |
| 96 | assert self.output_type != "generate_until" |
| 97 | |
| 98 | if "temperature" in self.generation_kwargs: |
| 99 | self.generation_kwargs["temperature"] = float( |
| 100 | self.generation_kwargs["temperature"] |
| 101 | ) |
| 102 | |