Function eval

paperflow/cli.py:418–453

Run the public PaperFlow-Bench evaluator on a predictions file.

(
    benchmark_dir: Path = typer.Option(
        Path("data/PaperFlow-Bench"),
        "--benchmark-dir",
        "-b",
        help="Benchmark root containing data/, reference_outputs/, evaluation/.",
    ),
    predictions: Path = typer.Option(
        ...,
        "--predictions",
        "-p",
        help="Top-20 predictions JSONL file (one row per episode).",
    ),
    output: Path = typer.Option(
        Path("paperflow_eval.json"),
        "--output",
        "-o",
        help="Where to write the metrics JSON.",
    ),
)

@app.command()
def eval(
    benchmark_dir: Path = typer.Option(
        Path("data/PaperFlow-Bench"),
        "--benchmark-dir",
        "-b",
        help="Benchmark root containing data/, reference_outputs/, evaluation/.",
    ),
    predictions: Path = typer.Option(
        ...,
        "--predictions",
        "-p",
        help="Top-20 predictions JSONL file (one row per episode).",
    ),
    output: Path = typer.Option(
        Path("paperflow_eval.json"),
        "--output",
        "-o",
        help="Where to write the metrics JSON.",
    ),
) -> None:
    """Run the public PaperFlow-Bench evaluator on a predictions file."""
    script = PROJECT_ROOT / "experiments" / "benchmark" / "evaluate_benchmark_predictions.py"
    if not script.exists():
        typer.echo(f"[error] evaluator not found: {script}", err=True)
        raise typer.Exit(code=1)
    raise typer.Exit(
        code=_run_python(
            script,
            "--benchmark-dir",
            str(benchmark_dir),
            "--predictions",
            str(predictions),
            "--output",
            str(output),
        )
    )


def main() -> None:  # pragma: no cover - thin wrapper