| 33 | VERSION = 0 |
| 34 | |
| 35 | def __init__(self, json_path): |
| 36 | self._random_seed = 42 |
| 37 | with open(json_path) as file: |
| 38 | self._task_json = json.load(file) |
| 39 | self._has_multi_choice = "multiple_choice_grade" in self._task_json["metrics"] |
| 40 | self._has_generative = "exact_str_match" in self._task_json["metrics"] |
| 41 | self.output_regex = self._task_json.get("output_regex", None) |
| 42 | self.stop_string = self._task_json.get("stop_string", None) |
| 43 | if self.output_regex is None and self.stop_string is None: |
| 44 | self.output_regex = _DEFAULT_REGEX |
| 45 | # differs from the default 30 when evaluating HF models in the BIG-bench codebase |
| 46 | self.max_length = 128 |
| 47 | |
| 48 | def has_training_docs(self): |
| 49 | return False |