| 137 | return questions, answers |
| 138 | |
| 139 | class MmluProDatasetHandler(DatasetHandler): |
| 140 | def __init__(self, answer_pattern: str = ANSWER_PATTERN_MULTICHOICE): |
| 141 | super().__init__(answer_pattern) |
| 142 | |
| 143 | def load_data(self): |
| 144 | dataset = load_dataset('TIGER-Lab/MMLU-Pro', split='test') |
| 145 | examples = [] |
| 146 | for row in dataset: |
| 147 | example = { |
| 148 | 'question': row['question'], |
| 149 | 'options': row['options'], |
| 150 | 'answer': row['answer'], |
| 151 | 'answer_index': row['answer_index'], |
| 152 | 'category': row['category'], |
| 153 | 'cot_content': row['cot_content'], |
| 154 | 'src': row['src'] |
| 155 | } |
| 156 | examples.append(example) |
| 157 | random.shuffle(examples) |
| 158 | examples = examples[:1000] |
| 159 | questions = [] |
| 160 | answers = [] |
| 161 | for example in examples: |
| 162 | # Format question with options |
| 163 | question = example['question'] + "\n\nOptions:\n" |
| 164 | for i, opt in enumerate(example['options']): |
| 165 | question += f"{chr(65+i)}. {opt}\n" |
| 166 | |
| 167 | questions.append(question) |
| 168 | answers.append(example['answer']) |
| 169 | |
| 170 | return questions, answers |
| 171 | |
| 172 | class bbehDatasetHandler(DatasetHandler): |
| 173 | def __init__(self, answer_pattern: str = ANSWER_PATTERN_BOXED): |
no outgoing calls
no test coverage detected