(third_task_data,MODEL=None)
| 263 | |
| 264 | |
| 265 | def RoadChange_criterion_QA(third_task_data,MODEL=None): |
| 266 | ques_total_num = 0 |
| 267 | right_num = 0 |
| 268 | obey_insytruction = 0 |
| 269 | totol_improve_score = 0 |
| 270 | for d_ind, sample in enumerate(third_task_data): |
| 271 | reference = sample['reference'] |
| 272 | prediction = sample['prediction'] |
| 273 | scores_list = [] |
| 274 | for q_ind, pred in enumerate(prediction): |
| 275 | ques_nopath = ''.join(sample['questions'][q_ind].lower().split(';')[1:]) |
| 276 | tips = extract_options(ques_nopath) |
| 277 | pred = remove_symbols(pred) |
| 278 | ques_total_num += 1 |
| 279 | clean_pred = clean_string(pred).lower() |
| 280 | options_nums = clean_pred.split("', '") |
| 281 | reference_q_ind = convert_if_number(reference[q_ind]).lower() |
| 282 | if len(options_nums)==1: |
| 283 | if clean_pred in ques_nopath: |
| 284 | obey_insytruction+=1 |
| 285 | if clean_pred==reference_q_ind: |
| 286 | right_num+=1 |
| 287 | scores_list.append(1) |
| 288 | elif reference_q_ind in clean_pred: |
| 289 | ### filter |
| 290 | if reference_q_ind in tips: |
| 291 | tips.remove(reference_q_ind) |
| 292 | if not any(tip in clean_pred for tip in tips): |
| 293 | right_num+=1 |
| 294 | scores_list.append(1) |
| 295 | else: |
| 296 | scores_list.append(0) |
| 297 | else: |
| 298 | scores_list.append(0) |
| 299 | else: |
| 300 | scores_list.append(0) |
| 301 | scores_list = np.array(scores_list) |
| 302 | scores = compare_and_count(scores_list[len(scores_list)//2:], scores_list[:len(scores_list)//2]) |
| 303 | totol_improve_score += scores |
| 304 | |
| 305 | return ques_total_num,right_num/ques_total_num,obey_insytruction/ques_total_num,totol_improve_score*2/ques_total_num |
| 306 | |
| 307 | def RoadSpeed_criterion_QA(third_task_data,MODEL=None): |
| 308 | ques_total_num = 0 |
nothing calls this directly
no test coverage detected