Function Relation_criterion_QA

evaluate_utils.py:206–261

(third_task_data,MODEL=None)

        
def Relation_criterion_QA(third_task_data,MODEL=None):
    ques_total_num = 0
    total_score = 0
    obey_insytruction = 0
    totol_improve_score = 0
    for d_ind, sample in enumerate(third_task_data):
        reference = sample['reference']
        prediction = sample['prediction']
        scores_list = []
        for q_ind, pred in enumerate(prediction):
            ques_total_num+=1
            if 'corresponds to' in pred:
                # pattern = r'(?<!\d)(-?\d+|[0-9]+/[0-9]+)(?!\d)'
                pattern = r'corresponds to No.([+-]?\d+|[+-]?\d+/\d+)'
                match = re.search(pattern, pred)
                if match:
                    pred_num = match.group(1).split('/')
                    # print(pred_num)
                else:
                    pred_num = []
            elif 'corresponding to' in pred:
                pattern = r"corresponding to.*is\s+(-?\d+(?:/\d+)*)"
                match = re.search(pattern, pred)
                if match:
                    pred_num = match.group(1).split("/")
                else:
                    pred_num = []
            else:
                pattern = r"(-?\d+(?:/\d+)*)"
                match = re.findall(pattern, pred)
                if match:
                    obey_insytruction+=1
                    pred_num = match[-1].split("/")
                    # print(pred_num)
                else:
                    pred_num = []
            
            ref_num =  reference[q_ind].split('/')
            if any(p_num not in ref_num for p_num in pred_num):
                scores_list.append(0)
                continue
            else:
                temp = 0
                # for p_num in pred_num:
                #     if p_num in ref_num:
                #         total_score += 1
                #         break
                for p_num in pred_num:
                    if p_num in ref_num:
                        temp += 1/len(ref_num)
                        total_score += 1/len(ref_num)
                scores_list.append(temp)
        scores_list = np.array(scores_list)
        scores = compare_and_count(scores_list[len(scores_list)//2:], scores_list[:len(scores_list)//2])
        totol_improve_score += scores
    return ques_total_num,total_score/ques_total_num,obey_insytruction/ques_total_num,totol_improve_score*2/ques_total_num