| 117 | |
| 118 | @staticmethod |
| 119 | def load_single(path, data_filename, label_filename): |
| 120 | data_path = os.path.join(path, data_filename) |
| 121 | label_path = os.path.join(path, label_filename) |
| 122 | dataset = [] |
| 123 | with open(data_path, 'r', encoding='utf-8') as f: |
| 124 | data_lines = f.readlines() |
| 125 | with open(label_path, 'r', encoding='utf-8') as f: |
| 126 | label_lines = f.readlines() |
| 127 | assert len(data_lines) == len(label_lines) |
| 128 | for data, label in zip(data_lines, label_lines): |
| 129 | i = json.loads(data.strip()) |
| 130 | i['label'] = int(label.strip()) |
| 131 | # some preprocessing |
| 132 | i['goal'] = i['goal'][0].upper() + i['goal'][1:] |
| 133 | if i['goal'].endswith('?') or i['goal'].endswith('.'): |
| 134 | i['sol1'] = i['sol1'][0].upper() + i['sol1'][1:] |
| 135 | i['sol2'] = i['sol2'][0].upper() + i['sol2'][1:] |
| 136 | else: |
| 137 | i['sol1'] = i['sol1'][0].lower() + i['sol1'][1:] |
| 138 | i['sol2'] = i['sol2'][0].lower() + i['sol2'][1:] |
| 139 | del i['id'] |
| 140 | dataset.append(i) |
| 141 | |
| 142 | return Dataset.from_list(dataset) |
| 143 | |
| 144 | @staticmethod |
| 145 | def load(path): |