Method __init__

numpy_ml/rl_models/agents.py:102–162

r""" A cross-entropy method agent. Notes ----- The cross-entropy method [1]_ [2]_ agent only operates on ``envs`` with discrete action spaces. On each episode the agent generates `n_theta_samples` of the parameters (:math:`\theta`) for its be

(self, env, n_samples_per_episode=500, retain_prcnt=0.2)

class CrossEntropyAgent(AgentBase):
    def __init__(self, env, n_samples_per_episode=500, retain_prcnt=0.2):
        r"""
        A cross-entropy method agent.

        Notes
        -----
        The cross-entropy method [1]_ [2]_ agent only operates on ``envs`` with
        discrete action spaces.

        On each episode the agent generates `n_theta_samples` of the parameters
        (:math:`\theta`) for its behavior policy. The `i`'th sample at
        timestep `t` is:

        .. math::

            \theta_i  &=  \{\mathbf{W}_i^{(t)}, \mathbf{b}_i^{(t)} \} \\
            \theta_i  &\sim  \mathcal{N}(\mu^{(t)}, \Sigma^{(t)})

        Weights (:math:`\mathbf{W}_i`) and bias (:math:`\mathbf{b}_i`) are the
        parameters of the softmax policy:

        .. math::

            \mathbf{z}_i  &=  \text{obs} \cdot \mathbf{W}_i + \mathbf{b}_i \\
            p(a_i^{(t + 1)})  &=  \frac{e^{\mathbf{z}_i}}{\sum_j e^{z_{ij}}} \\
            a^{(t + 1)}  &=  \arg \max_j p(a_j^{(t+1)})

        At the end of each episode, the agent takes the top `retain_prcnt`
        highest scoring :math:`\theta` samples and combines them to generate
        the mean and variance of the distribution of :math:`\theta` for the
        next episode:

        .. math::

            \mu^{(t+1)}  &=  \text{avg}(\texttt{best_thetas}^{(t)}) \\
            \Sigma^{(t+1)}  &=  \text{var}(\texttt{best_thetas}^{(t)})

        References
        ----------
        .. [1] Mannor, S., Rubinstein, R., & Gat, Y. (2003). The cross entropy
           method for fast policy search. In *Proceedings of the 20th Annual
           ICML, 20*.
        .. [2] Rubinstein, R. (1997). optimization of computer simulation
           models with rare events, *European Journal of Operational Research,
           99*, 89–112.

        Parameters
        ----------
        env : :meth:`gym.wrappers` or :meth:`gym.envs` instance
            The environment to run the agent on.
        n_samples_per_episode : int
            The number of theta samples to evaluate on each episode. Default is 500.
        retain_prcnt: float
            The percentage of `n_samples_per_episode` to use when calculating
            the parameter update at the end of the episode. Default is 0.2.
        """
        super().__init__(env)