(self, path: str, name: str, *args, **kwargs)
| 94 | class CsimpleqaDataset(BaseDataset): |
| 95 | |
| 96 | def load(self, path: str, name: str, *args, **kwargs): |
| 97 | path = get_data_path(path) |
| 98 | filename = osp.join(path, f'{name}.jsonl') |
| 99 | dataset = DatasetDict() |
| 100 | raw_data = [] |
| 101 | lines = open(filename, 'r', encoding='utf-8').readlines() |
| 102 | for line in lines: |
| 103 | data = json.loads(line) |
| 104 | question = data['question'] |
| 105 | cur_system_prompt = '你是一个智能助手。' |
| 106 | messages = [{ |
| 107 | 'role': 'system', |
| 108 | 'content': cur_system_prompt |
| 109 | }, { |
| 110 | 'role': 'user', |
| 111 | 'content': question |
| 112 | }] |
| 113 | judge_system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。' |
| 114 | csimpleqa_judge_prompt_f = csimpleqa_judge_prompt_new.format( |
| 115 | question=question, |
| 116 | target=data['answer'], |
| 117 | predicted_answer='{prediction}') |
| 118 | raw_data.append({ |
| 119 | 'primary_category': data['primary_category'], |
| 120 | 'question': question, |
| 121 | 'gold_ans': data['answer'], |
| 122 | 'messages': messages, |
| 123 | 'system_prompt': judge_system_prompt, |
| 124 | 'prompt_template': csimpleqa_judge_prompt_f, |
| 125 | 'judge': { |
| 126 | 'primary_category': data['primary_category'], |
| 127 | 'question': question, |
| 128 | 'question_id': data['id'] |
| 129 | } |
| 130 | }) |
| 131 | dataset = Dataset.from_list(raw_data) |
| 132 | return dataset |
| 133 | |
| 134 | |
| 135 | def post_process_csimpleqa(completion): |
nothing calls this directly
no test coverage detected