CLI e2e: run sol-execbench on a GQA paged-decode problem with safetensors inputs.
(tmp_path: Path)
| 307 | |
| 308 | @pytest.mark.xdist_group("serial") |
| 309 | def test_cli_gqa_paged_decode(tmp_path: Path): |
| 310 | """CLI e2e: run sol-execbench on a GQA paged-decode problem with safetensors inputs.""" |
| 311 | sample_dir = _SAMPLES_DIR / "gqa_paged_decode" |
| 312 | output_file = tmp_path / "traces.jsonl" |
| 313 | |
| 314 | result = subprocess.run( |
| 315 | [ |
| 316 | "uv", |
| 317 | "run", |
| 318 | "sol-execbench", |
| 319 | str(sample_dir), |
| 320 | "-o", |
| 321 | str(output_file), |
| 322 | ], |
| 323 | capture_output=True, |
| 324 | text=True, |
| 325 | timeout=300, |
| 326 | ) |
| 327 | assert result.returncode == 0, ( |
| 328 | f"CLI failed:\n stdout={result.stdout}\n stderr={result.stderr}" |
| 329 | ) |
| 330 | |
| 331 | assert output_file.exists(), "Output file not created" |
| 332 | lines = [l for l in output_file.read_text().splitlines() if l.strip()] |
| 333 | assert len(lines) == 2, f"Expected 2 traces, got {len(lines)}" |
| 334 | |
| 335 | for line in lines: |
| 336 | trace = Trace(**json.loads(line)) |
| 337 | assert trace.evaluation.status == EvaluationStatus.PASSED, ( |
| 338 | f"Workload {trace.workload.uuid} did not pass: " |
| 339 | f"status={trace.evaluation.status.value} log={trace.evaluation.log}" |
| 340 | ) |