| 30 | |
| 31 | |
| 32 | class BigBenchJsonTask(Task): |
| 33 | VERSION = 0 |
| 34 | |
| 35 | def __init__(self, json_path): |
| 36 | self._random_seed = 42 |
| 37 | with open(json_path) as file: |
| 38 | self._task_json = json.load(file) |
| 39 | self._has_multi_choice = "multiple_choice_grade" in self._task_json["metrics"] |
| 40 | self._has_generative = "exact_str_match" in self._task_json["metrics"] |
| 41 | self.output_regex = self._task_json.get("output_regex", None) |
| 42 | self.stop_string = self._task_json.get("stop_string", None) |
| 43 | if self.output_regex is None and self.stop_string is None: |
| 44 | self.output_regex = _DEFAULT_REGEX |
| 45 | # differs from the default 30 when evaluating HF models in the BIG-bench codebase |
| 46 | self.max_length = 128 |
| 47 | |
| 48 | def has_training_docs(self): |
| 49 | return False |
| 50 | |
| 51 | def has_validation_docs(self): |
| 52 | return False |
| 53 | |
| 54 | def has_test_docs(self): |
| 55 | return True |
| 56 | |
| 57 | def test_docs(self): |
| 58 | return _get_unique_examples(self._task_json["examples"]) |
| 59 | |
| 60 | def doc_to_text(self, doc): |
| 61 | example_input_prefix = self._task_json.get("example_input_prefix", "\nQ: ") |
| 62 | res = f"{example_input_prefix}{doc['input']}" |
| 63 | |
| 64 | rng = np.random.RandomState(seed=self._random_seed) |
| 65 | choice_prefix = self._task_json.get("choice_prefix", "\n choice: ") |
| 66 | append_choices = self._task_json.get("append_choices_to_input", True) |
| 67 | if "target_scores" in doc and append_choices: |
| 68 | choice_dict = doc["target_scores"] |
| 69 | permuted_choices = rng.permutation(sorted(list(choice_dict.keys()))) |
| 70 | res = f"{res}{choice_prefix}{choice_prefix.join(permuted_choices)}" |
| 71 | |
| 72 | example_output_prefix = self._task_json.get("example_output_prefix", "\nA: ") |
| 73 | res = f"{res}{example_output_prefix}" |
| 74 | return res |
| 75 | |
| 76 | def doc_to_target(self, doc): |
| 77 | return max(doc["target_scores"].items(), key=lambda x: x[1])[0] |
| 78 | |
| 79 | def _doc_to_queries(self, doc): |
| 80 | if "target_scores" in doc: |
| 81 | return list(doc["target_scores"].keys()) |
| 82 | return doc["target"] if isinstance(doc["target"], list) else [doc["target"]] |
| 83 | |
| 84 | def construct_requests(self, doc, ctx): |
| 85 | requests = [] |
| 86 | if self._has_multi_choice: |
| 87 | queries = self._doc_to_queries(doc) |
| 88 | requests += [ |
| 89 | rf.loglikelihood(ctx, continuation)[0] for continuation in queries |
nothing calls this directly
no outgoing calls
no test coverage detected