Function evaluate

4-atari-hard/3-robustify.py:335–368

From-reset, sticky-action, eps-greedy 0.0 eval — the RL-policy number.

(net, demo, device, n_episodes, n_actions, seed)

def evaluate(net, demo, device, n_episodes, n_actions, seed):
    """From-reset, sticky-action, eps-greedy 0.0 eval — the RL-policy number."""
    e = ReplayResetEnv(demo, seed=seed + 99, sticky=STICKY, noop_max=30)
    e.starting_point = 0  # always from reset
    e.frac_sample = 0.0
    # Eval must honor targets montezuma_goexplore_robust.protocol.termination=game_over:
    # turn OFF the training-curriculum kills so a from-reset episode runs to a real
    # game_over, not a ~allowed_lag-step lag-kill window. Otherwise value_mean reports a
    # key-but-slower-than-demo policy as ~0 and the canary's first-key/retreat signal is
    # destroyed (preflight S1). lag-kill needs t>allowed_lag & t<n -> allowed_lag=n makes
    # it unreachable; success-kill needs score>=total_return-deficit -> huge -deficit off.
    e.allowed_lag = e.n
    e.allowed_score_deficit = -1e18
    e.max_steps = 4500  # standard Montezuma eval cap = 18000 frames / frameskip 4
    #                     (atari-ale-protocol; same as RND montezuma episode_cap). Bounds
    #                     a passive policy that would otherwise never reach game_over.
    scores = []
    for _ in range(n_episodes):
        frame = e.reset()
        stack = _stack_init(frame)
        hx = torch.zeros(1, net.gru_dim, device=device)
        ret = 0.0
        done = False
        while not done:
            with torch.no_grad():
                obs = torch.as_tensor(stack[None], dtype=torch.float32, device=device)
                logits, _, hx = net.step(obs, hx)
                a = int(logits.argmax(-1))
            frame, _, done, info = e.step(a)
            ret += info["raw_reward"]
            stack = np.concatenate([stack[1:], frame[None]], axis=0)
        scores.append(ret)
    e.env.close()
    return scores


if __name__ == "__main__":