| 19 | |
| 20 | |
| 21 | class Dataset(object): |
| 22 | def __init__(self, name, data_path = None, testfile_path = None, full = True, train_description_path = None, tmp_dir = "./tmp") -> None: |
| 23 | self.name = name |
| 24 | self.index = -1 |
| 25 | self.tmp_dir = tmp_dir |
| 26 | self.data_path = data_path |
| 27 | if not data_path: |
| 28 | self.dataset = load_dataset(name, cache_dir = "./datasets") |
| 29 | else: |
| 30 | self.dataset = datasets.load_from_disk(data_path) |
| 31 | |
| 32 | if testfile_path and name == "NTU-NLP-sg/xCodeEval": |
| 33 | self.testcases = json.load(open(testfile_path, "r")) |
| 34 | if train_description_path != None: |
| 35 | self.descriptions = json.load(open(train_description_path, "r")) |
| 36 | elif testfile_path and name in ["mbpp", "openai_humaneval"]: |
| 37 | raw_data = open(testfile_path, "r").read().splitlines() |
| 38 | data = [] |
| 39 | for line in raw_data: |
| 40 | data.append(json.loads(line)) |
| 41 | self.testcases = {} |
| 42 | for instance in data: |
| 43 | if self.name == "mbpp": |
| 44 | self.testcases[instance["task_id"].replace("HumanEval/", "").replace("Mbpp/", "")] = {"inputs": instance["base_input"], "outputs": None, "entry_point": instance["entry_point"]} |
| 45 | else: |
| 46 | self.testcases[instance["task_id"].replace("HumanEval/", "").replace("Mbpp/", "")] = {"inputs": instance["base_input"], "outputs": None} |
| 47 | else: |
| 48 | self.testcases = {} |
| 49 | |
| 50 | |
| 51 | self.code_keywords = { |
| 52 | "openai_humaneval": "canonical_solution", |
| 53 | "codeparrot/apps": "solutions", |
| 54 | "mbpp": "code", |
| 55 | "deepmind/code_contests": "solutions", |
| 56 | "BAAI/TACO": "solutions", |
| 57 | "NTU-NLP-sg/xCodeEval": None |
| 58 | } |
| 59 | |
| 60 | self.testcase_keywords = { |
| 61 | "openai_humaneval": "testcases", |
| 62 | "codeparrot/apps": "input_output", |
| 63 | "mbpp": "testcases", |
| 64 | "deepmind/code_contests": ["private_tests", "generated_tests"], |
| 65 | "BAAI/TACO": "input_output", |
| 66 | "NTU-NLP-sg/xCodeEval": "testcases" |
| 67 | } |
| 68 | |
| 69 | self.add_list = { |
| 70 | "openai_humaneval": False, |
| 71 | "codeparrot/apps": True, |
| 72 | "mbpp": False, |
| 73 | "deepmind/code_contests": True, |
| 74 | "BAAI/TACO": True, |
| 75 | "NTU-NLP-sg/xCodeEval": True, |
| 76 | } |
| 77 | |
| 78 | if full: |
no outgoing calls
no test coverage detected