(path: str, prompt_mode: str, **kwargs)
| 26 | |
| 27 | @staticmethod |
| 28 | def load(path: str, prompt_mode: str, **kwargs): |
| 29 | |
| 30 | def capitalize_first_letter(s): |
| 31 | if not s: # 检查字符串是否为空 |
| 32 | return s |
| 33 | return s[0].upper() + s[1:] |
| 34 | |
| 35 | subset = kwargs['subset'] |
| 36 | data_files = {} |
| 37 | test_file = f'data/{capitalize_first_letter(subset)}/' |
| 38 | test_file += f'sciknoweval_{subset}_test.jsonl' |
| 39 | data_files['test'] = test_file |
| 40 | dataset = load_dataset(path, data_files=data_files, split='test') |
| 41 | # dataset = dataset.select(range(20)) |
| 42 | if prompt_mode == 'zero-shot': |
| 43 | dataset = dataset.map( |
| 44 | lambda item: _parse(item, prompt_mode, subset), |
| 45 | load_from_cache_file=False) |
| 46 | elif prompt_mode == 'few-shot': |
| 47 | pass # TODO: Implement few-shot prompt |
| 48 | return dataset |
| 49 | |
| 50 | |
| 51 | class SciKnowEvalEvaluator(BaseEvaluator): |
nothing calls this directly
no test coverage detected