Function main

tests/test.py:145–186

()

                print(f"     Error: {approach_result['result']}")

def main():
    parser = argparse.ArgumentParser(description="Test different LLM inference approaches.")
    parser.add_argument("--test_cases", type=str, default=None, help="Path to test cases JSON file")
    parser.add_argument("--approaches", nargs='+', default=list(APPROACHES.keys()), help="Approaches to test")
    parser.add_argument("--model", type=str, default="gpt-4o-mini", help="Model to use for testing")
    parser.add_argument("--base-url", type=str, default=None, help="The base_url for the OpenAI API compatible endpoint")
    parser.add_argument("--single-test", type=str, default=None, help="Name of a single test case to run")
    args = parser.parse_args()
    
    # Set default test_cases path relative to this script
    if args.test_cases is None:
        script_dir = os.path.dirname(os.path.abspath(__file__))
        args.test_cases = os.path.join(script_dir, "test_cases.json")
    
    # If using local inference mode, override model to a local model
    if API_KEY == "optillm" and args.model == "gpt-4o-mini":
        args.model = TEST_MODEL
        logger.info(f"Using local model: {args.model}")
    
    # Set environment variable for local inference
    if API_KEY == "optillm":
        os.environ["OPTILLM_API_KEY"] = "optillm"

    test_cases = load_test_cases(args.test_cases)

    # Use local inference by default for testing
    if args.base_url:
        client = OpenAI(api_key=API_KEY, base_url=args.base_url)
    elif API_KEY == "optillm":
        # Use local inference endpoint
        client = OpenAI(api_key=API_KEY, base_url="http://localhost:8000/v1")
        logger.info("Using local inference endpoint: http://localhost:8000/v1")
    else:
        client = OpenAI(api_key=API_KEY)
        # client = LiteLLMWrapper()

    results = run_tests(test_cases, args.approaches, client, args.model, args.single_test)
    print_summary(results)

    # Optionally, save detailed results to a file
    with open('test_results.json', 'w') as f:
        json.dump(results, f, indent=2)

if __name__ == "__main__":
    main()