(self, result: Tuple[Dict[str, Any], Any], data: Dict[str, Any])
| 109 | return "\n".join(parts) |
| 110 | |
| 111 | def construct_entry(self, result: Tuple[Dict[str, Any], Any], data: Dict[str, Any]) -> Dict[str, Any]: |
| 112 | # Construct result entry |
| 113 | result_data, log = result |
| 114 | question_key = self.config.question_key |
| 115 | answer_key = self.config.answer_key |
| 116 | |
| 117 | # Handle case where question_key is a list |
| 118 | if isinstance(question_key, list): |
| 119 | question = self._format_question_from_keys(data, question_key) |
| 120 | else: |
| 121 | question = data[question_key] |
| 122 | |
| 123 | groundtruth = data[answer_key] |
| 124 | |
| 125 | entry = { |
| 126 | "problem": question, |
| 127 | "groundtruth": groundtruth, |
| 128 | "response": result_data.get("response"), |
| 129 | "answer": result_data.get("answer"), |
| 130 | "log": log |
| 131 | } |
| 132 | |
| 133 | # Dynamically import scoring function |
| 134 | scoring_function = getattr(__import__(f"experiment.utils", fromlist=[self.config.scoring_function]), |
| 135 | self.config.scoring_function) |
| 136 | |
| 137 | # Pass different parameters based on scoring function |
| 138 | if self.config.scoring_function == "score_math": |
| 139 | entry["score"] = scoring_function(entry["answer"], groundtruth, self.dataset) |
| 140 | else: |
| 141 | entry["score"] = scoring_function(entry["answer"], groundtruth) |
| 142 | return entry |
| 143 | |
| 144 | def save_final_info(self, accuracy: float) -> None: |
| 145 | final_info_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "final_info.json") |
no test coverage detected