Function main

scripts/eval_math500_benchmark.py:745–782

Main evaluation function.

(model: str)

            print("---")

def main(model: str):
    """Main evaluation function."""
    os.makedirs("results", exist_ok=True)
    results_file = f"evaluation_results_math500_{model.replace('/', '_')}.json"
    
    dataset = load_math500_dataset()
    existing_results = load_existing_results(results_file)
    
    # Create a set of already processed indexes for efficient lookup
    processed_indexes = {result['index'] for result in existing_results}
    
    for idx, item in enumerate(tqdm(dataset, desc="Evaluating problems")):
        # Skip if this index has already been processed
        if idx in processed_indexes:
            continue
            
        problem_text = item['problem']
        correct_answer = item['answer']
        
        # Get model's response
        response = get_llm_response(problem_text, model)
        predicted_answer = extract_answer(response)

        # Compare answers using the new comparison function
        is_correct = compare_answers(correct_answer, predicted_answer)
        
        result = {
            "index": idx,
            "problem": problem_text,
            "response": response,
            "correct_answer": correct_answer,
            "predicted_answer": predicted_answer,
            "is_correct": is_correct
        }
        save_result(results_file, result)
    
    final_results = load_existing_results(results_file)
    analyze_results(final_results)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Evaluate LLM performance on MATH-500 problems")