Function main

scripts/run_dataset.py:346–563

()

def main():
    ap = argparse.ArgumentParser(
        description="Run SOL ExecBench problems using reference implementations.",
    )
    ap.add_argument(
        "problems_dir",
        type=Path,
        help="Path to a single problem directory (contains definition.json + workload.jsonl) "
        "or a dataset root with category sub-directories (e.g. L1/, L2/).",
    )
    ap.add_argument(
        "--category",
        type=str,
        nargs="+",
        default=None,
        choices=sorted(CATEGORIES),
        help="Restrict to one or more categories (e.g. --category L1 L2).",
    )
    ap.add_argument(
        "--limit",
        type=int,
        default=None,
        help="Max number of problems to evaluate.",
    )
    ap.add_argument(
        "-o",
        "--output",
        type=Path,
        default=ROOT / "out",
        help="Output directory for traces and summary. Defaults to <repo_root>/out.",
    )
    ap.add_argument(
        "--timeout",
        type=int,
        default=300,
        help="Per-problem GPU evaluation timeout in seconds (default: 300).",
    )
    ap.add_argument(
        "--max-workloads",
        type=int,
        default=None,
        help="Max number of workloads per problem. Truncates the workload file if exceeded.",
    )
    ap.add_argument(
        "--iterations",
        type=int,
        default=None,
        help="Number of timing iterations per workload (default: 50, from BenchmarkConfig).",
    )
    ap.add_argument(
        "--solution-name",
        type=str,
        default=None,
        help="Filename to look for in each problem directory as the solution "
        "(e.g. solution.py, solution.json). "
        ".py files are wrapped into a solution JSON automatically; "
        ".json files are loaded directly. "
        "Defaults to None (uses definition.reference).",