()
| 65 | logging.basicConfig(level=level, format='%(asctime)s - %(levelname)s - %(message)s') |
| 66 | |
| 67 | def main(): |
| 68 | args = parse_args() |
| 69 | # set_up_logging() |
| 70 | # Ensure at least one task is selected |
| 71 | if not (args.generate or args.verify or args.evaluate): |
| 72 | print("Please select at least one task (--generate, --verify, or --evaluate)") |
| 73 | return |
| 74 | |
| 75 | # Step 0: Download datasets |
| 76 | if args.auto_dl: |
| 77 | from datasets import load_dataset |
| 78 | |
| 79 | if args.datasets == "FomaMATH-All": |
| 80 | input_dataset_id = "SphereLab/FormalMATH-All" |
| 81 | elif args.datasets == "FomaMATH-Lite": |
| 82 | input_dataset_id = "SphereLab/FormalMATH-Lite" |
| 83 | else: |
| 84 | raise ValueError(f"Unknown dataset: {args.datasets}") |
| 85 | |
| 86 | input_dataset_branch = "main" |
| 87 | local_dataset_path = "./data/" |
| 88 | |
| 89 | os.makedirs(local_dataset_path, exist_ok=True) |
| 90 | |
| 91 | try: |
| 92 | args.input_file = os.path.join(local_dataset_path, "FomaMATH.json") |
| 93 | args.generated_file = os.path.join(local_dataset_path, "FomaMATH_generated.json") |
| 94 | args.verification_file = os.path.join(local_dataset_path, "FomaMATH_verification.json") |
| 95 | args.evaluation_file = os.path.join(local_dataset_path, "FomaMATH_evaluation.json") |
| 96 | |
| 97 | ds = load_dataset(input_dataset_id, split="train", revision=input_dataset_branch) |
| 98 | ds.to_json(args.input_file) |
| 99 | print(f"Dataset has been saved to: {local_dataset_path}") |
| 100 | |
| 101 | except Exception as e: |
| 102 | print(f"Error occurred while downloading dataset: {e}") |
| 103 | |
| 104 | # Step 1: Generate answers |
| 105 | if args.generate: |
| 106 | try: |
| 107 | print(f"Generating answers using model {args.model}") |
| 108 | process_data( |
| 109 | model_path=args.model, |
| 110 | input_file=args.input_file, |
| 111 | output_file=args.generated_file, |
| 112 | batch_size=args.n, |
| 113 | num_answers=args.nums_answer |
| 114 | ) |
| 115 | print(f"Answers have been generated and saved to {args.generated_file}") |
| 116 | except Exception as e: |
| 117 | logging.error(f"Error during answer generation: {e}") |
| 118 | return |
| 119 | |
| 120 | # Step 2: Verify answers |
| 121 | if args.verify: |
| 122 | try: |
| 123 | print("Starting verification of answers") |
| 124 | # Use the generated file as input if answers were generated, otherwise use the provided input file |
no test coverage detected