Method load

opencompass/datasets/mathbench.py:115–145

(path: str, name: str, with_circular: bool = True)

    @staticmethod
    def load(path: str, name: str, with_circular: bool = True):
        data = []
        filename = osp.join(path, f'{name}.jsonl')
        with open(filename, 'r', encoding='utf-8') as infile:
            for id, line in enumerate(infile):
                entry = json.loads(line)
                if 'cloze' in name:
                    data.append({
                        'question': entry['question'].strip(),
                        'answer': entry['answer'].strip()
                    })
                else:
                    if with_circular:
                        data.extend(get_circular_example(entry, id))
                    else:
                        question = entry['question'].strip(
                        ) + '\n' + get_number(entry['options'])
                        info = {
                            'question': question,
                            'answer': entry['answer'].strip()
                        }
                        # For PPL evaluation
                        for i in range(4):
                            info[chr(ord('A') +
                                     i)] = entry['options'][i].strip()
                        data.append(info)

        if 'cloze' not in name:
            data = data[:(len(data) // 4 + 7) // 8 * 8]
        dataset = Dataset.from_list(data)
        return dataset


import collections