| 97 | |
| 98 | |
| 99 | def verify(f, sessions, config): |
| 100 | verified = [] |
| 101 | for row in sessions: |
| 102 | sid = row["session"] |
| 103 | ledger = json.loads(f.run("session", "show", sid, "--json").stdout)[1] |
| 104 | assert len(ledger) == config["turns"], (sid, ledger) |
| 105 | for turn, entry in enumerate(ledger, 1): |
| 106 | state = f.rpc({"TurnStatus": {"session_id": sid, "turn_number": turn}})["TurnLifecycle"]["turn"] |
| 107 | assert state["state"] == "Completed", state |
| 108 | attempt = state["checkpoint_attempt"] |
| 109 | assert attempt["phase"] == "Published", attempt |
| 110 | assert not entry["change_hashes"], entry |
| 111 | assert entry["provenance_hash"] == attempt["provenance_hash"] |
| 112 | frozen = f.rpc({"LoadFrozenEnvelopes": {"provenance_id": state["provenance_id"]}})["FrozenEnvelopes"]["envelopes"] |
| 113 | envelopes = [json.loads(bytes(e)) for e in frozen] |
| 114 | assert all(e["session_id"] == sid and e["turn_number"] == turn for e in envelopes) |
| 115 | assert len({e["event_id"] for e in envelopes}) == len(envelopes) |
| 116 | events = [e for e in envelopes if e["event"].get("tool_call_id", "").startswith(f"bench-{turn}-")] |
| 117 | assert len(events) == config["events"], (sid, turn, len(events)) |
| 118 | assert len({e["event"]["tool_call_id"] for e in events}) == config["events"] |
| 119 | for event in events: |
| 120 | i = int(event["event"]["tool_call_id"].rsplit("-", 1)[1]) |
| 121 | assert event["event"]["output"] == base.output_payload(i, 1024) |
| 122 | assert event["event"]["raw"]["tool_output"] == base.output_payload(i, 1024) |
| 123 | digest = entry["provenance_hash"] |
| 124 | data = (f.root / ".atomic" / "changes" / digest[:2] / f"{digest}.provenance").read_bytes() |
| 125 | assert base64.b32encode(base.blake3.blake3(data).digest()).decode().rstrip("=") == digest |
| 126 | if turn > 1: |
| 127 | assert attempt["source"]["previous_provenance"] == ledger[turn - 2]["provenance_hash"] |
| 128 | f.hook("stop", {"session_id": sid, "reason": "end_turn", "response": "duplicate delivery"}) |
| 129 | assert json.loads(f.run("session", "show", sid, "--json").stdout)[1] == ledger |
| 130 | verified.append({"session": sid, "turns": len(ledger), "payloads_and_hashes": True, "duplicate_stop": True}) |
| 131 | return verified |
| 132 | |
| 133 | |
| 134 | def measure(binary, workers, config, template): |