Function parse_cot_eval_instruct

eval_kk/kk_processor.py:88–146

(pred_str, ans,
                   conclusion_patterns=['<answer>'],
                   verbose=False,
                   finish_patterns=["</answer>"],
                   reformat_gold_conditions=None,
                   expected_names=None,
                   solution_text_format=None)

def parse_cot_eval_instruct(pred_str, ans,
                   conclusion_patterns=['<answer>'],
                   verbose=False,
                   finish_patterns=["</answer>"],
                   reformat_gold_conditions=None,
                   expected_names=None,
                   solution_text_format=None):
    print("\n" + "="*80)
    print(" Processing New Sample ".center(80, '='))
    
    # Parse ground truth data
    gt_status = parse_solution_text_format(solution_text_format)
    expected_names = list(gt_status.keys())
    print(f"[Ground Truth] Final identities: {gt_status}")

    # Extract model answer
    answer_text, processed_str = extract_solution(pred_str)
    print(f"\n[Model Response]\n{processed_str}")

    # Validate response structure
    format_correct = validate_response_structure(processed_str)
    print(f"\n  Format validation: {'PASS' if format_correct else 'FAIL'}")

    # Validate answer content
    answer_score = 0
    is_correct = False
    correct_ratio = 0
    wrong_reason = "no_conclusion_matched"
    if format_correct and answer_text:
        pred_status = parse_model_answer(answer_text, expected_names)
        if pred_status:
            print(f"\n[Content Validation]")
            print(f"  Expected: {gt_status}")
            print(f"  Predicted: {pred_status}")
            
            if pred_status == gt_status:
                answer_score = 2
                is_correct = True
                correct_ratio = 1
                print("  Content validation: FULL MATCH")
            else:
                answer_score = -1.5
                correct_ratio = 0
                wrong_reason = "wrong_identity"
                print("  Content validation: MISMATCH")
        else:
            answer_score = -2
            correct_ratio = 0
            wrong_reason = "no_conclusion_matched"
            print( "Fail to parse answer")
    else:
        print("\n[Content Validation] Skipped due to format errors or missing answer")
    
    if is_correct == False and verbose == True:
        print("wrong_reason:",wrong_reason)
        print("********* \nprediction before parse:\n", pred_str)
        print("********* \nprediction after parse:\n", answer_text)