Function test_example

tests/examples/test_examples.py:196–264

End-to-end evaluation: load example -> package -> compile (if C++) -> execute -> assert PASSED.

(tmp_path: Path, case: Example)

    [pytest.param(c, id=c.test_id, marks=_mark_example(c)) for c in _EXAMPLES],
)
def test_example(tmp_path: Path, case: Example):
    """End-to-end evaluation: load example -> package -> compile (if C++) -> execute -> assert PASSED."""
    definition, solution, workloads = _load_example(
        case.language, case.problem, case.solution_file
    )
    config = BenchmarkConfig(lock_clocks=False)

    pkg = ProblemPackager(
        definition=definition,
        workloads=workloads,
        solution=solution,
        config=config,
        output_dir=tmp_path / "staging",
        keep_output_dir=True,
    )

    # Phase 1 (C++ only): compile
    languages = {lang.value for lang in solution.spec.languages}
    if languages & _CPP_LANGUAGES:
        cmd, artifact_path = pkg.compile()
        result = _run_subprocess(cmd, cwd=pkg.output_dir)
        assert result.returncode == 0, (
            f"Compilation failed for {case.test_id}:\n"
            f"  stdout={result.stdout}\n  stderr={result.stderr}"
        )
        assert Path(artifact_path).exists(), (
            f"benchmark_kernel.so not produced for {case.test_id}"
        )

    # Phase 2: GPU evaluation
    cmd = pkg.execute()
    result = _run_subprocess(cmd, cwd=pkg.output_dir)
    assert result.returncode == 0, (
        f"Execution failed for {case.test_id}:\n"
        f"  stdout={result.stdout}\n  stderr={result.stderr}"
    )

    traces = pkg.convert_stdout_to_traces(result.stdout)
    assert len(traces) == case.expected_count, (
        f"Expected {case.expected_count} traces for {case.test_id}, got {len(traces)}"
    )

    failed = [t for t in traces if not t.is_successful()]
    assert not failed, (
        f"{case.test_id}: {len(failed)}/{case.expected_count} workloads did not pass:\n"
        + "\n".join(
            f"  [{t.evaluation.status.value}] uuid={t.workload.uuid}  "
            f"log={t.evaluation.log}"
            for t in failed
        )
    )

    # Per-trace invariants for PASSED workloads
    for trace in traces:
        assert trace.definition == definition.name

        ev = trace.evaluation
        assert ev.correctness is not None, (