Method act

numpy_ml/rl_models/agents.py:195–236

r""" Generate actions according to a softmax policy. Notes ----- The softmax policy assumes that the pmf over actions in state :math:`x_t` is given by: .. math:: \pi(a | x^{(t)}) = \text{softmax}( \text{obs}^{(t)} \cdot \

(self, obs)

        self.episode_history = {"rewards": [], "state_actions": []}

    def act(self, obs):
        r"""
        Generate actions according to a softmax policy.

        Notes
        -----
        The softmax policy assumes that the pmf over actions in state :math:`x_t` is
        given by:

        .. math::

            \pi(a | x^{(t)}) = \text{softmax}(
                \text{obs}^{(t)} \cdot \mathbf{W}_i^{(t)} + \mathbf{b}_i^{(t)} )

        where :math:`\mathbf{W}` is a learned weight matrix, `obs` is the observation
        at timestep `t`, and **b** is a learned bias vector.

        Parameters
        ----------
        obs : int or :py:class:`ndarray <numpy.ndarray>`
            An observation from the environment.

        Returns
        -------
        action : int, float, or :py:class:`ndarray <numpy.ndarray>`
            An action sampled from the distribution over actions defined by the
            softmax policy.
        """
        E, P = self.env_info, self.parameters
        W, b = P["W"], P["b"]

        s = self._obs2num[obs]
        s = np.array([s]) if E["obs_dim"] == 1 else s

        # compute softmax
        Z = s.T @ W + b
        e_Z = np.exp(Z - np.max(Z, axis=-1, keepdims=True))
        action_probs = e_Z / e_Z.sum(axis=-1, keepdims=True)

        # sample action
        a = np.random.multinomial(1, action_probs).argmax()
        return self._num2action[a]

    def run_episode(self, max_steps, render=False):
        """