| 8 | |
| 9 | @dataclass |
| 10 | class CodeExecutionProblem: |
| 11 | question_id: str |
| 12 | contest_id: str |
| 13 | contest_date: datetime |
| 14 | difficulty: str |
| 15 | function_name: str |
| 16 | code: str |
| 17 | input: str |
| 18 | output: str |
| 19 | id: str |
| 20 | problem_id: str |
| 21 | numsteps: int |
| 22 | |
| 23 | def __post_init__(self): |
| 24 | pass |
| 25 | |
| 26 | def insert_output(self, output_list: list[str], pred_list: list[str]) -> dict: |
| 27 | return { |
| 28 | "question_id": self.question_id, |
| 29 | "contest_id": self.contest_id, |
| 30 | "contest_date": self.contest_date.isoformat(), |
| 31 | "difficulty": self.difficulty, |
| 32 | "function_name": self.function_name, |
| 33 | "code": self.code, |
| 34 | "input": self.input, |
| 35 | "output": self.output, |
| 36 | "id": self.id, |
| 37 | "problem_id": self.problem_id, |
| 38 | "numsteps": self.numsteps, |
| 39 | "output_list": output_list, |
| 40 | "pred_list": pred_list, |
| 41 | } |
| 42 | |
| 43 | def insert_output_evaluation( |
| 44 | self, output_list: list[str], code_list: list[str], graded_list: list[bool] |
| 45 | ) -> dict: |
| 46 | output = self.insert_output(output_list, code_list) |
| 47 | output["graded_list"] = graded_list |
| 48 | output["pass@1"] = graded_list.count(True) / len(graded_list) |
| 49 | return output |
| 50 | |
| 51 | def get_evaluation_sample(self) -> dict: |
| 52 | return { |
| 53 | "code": self.code, |
| 54 | "input": self.input, |
| 55 | "output": self.output, |
| 56 | } |
| 57 | |
| 58 | |
| 59 | def load_code_execution_dataset(release_version="release_v1") -> list[CodeExecutionProblem]: |
no outgoing calls
no test coverage detected