Method __init__

numpy_ml/bandits/policies.py:277–332

r""" A conjugate Thompson sampling [1]_ [2]_ policy for multi-armed bandits with Bernoulli likelihoods. Notes ----- The policy assumes independent Beta priors on the Bernoulli arm payoff probabilities, :math:`\theta`: .. math:: \

(self, alpha=1, beta=1)

class ThompsonSamplingBetaBinomial(BanditPolicyBase):
    def __init__(self, alpha=1, beta=1):
        r"""
        A conjugate Thompson sampling [1]_ [2]_ policy for multi-armed bandits with
        Bernoulli likelihoods.

        Notes
        -----
        The policy assumes independent Beta priors on the Bernoulli arm payoff
        probabilities, :math:`\theta`:

        .. math::

            \theta_k \sim \text{Beta}(\alpha_k, \beta_k) \\
            r \mid \theta_k \sim \text{Bernoulli}(\theta_k)

        where :math:`k \in \{1,\ldots,K \}` indexes arms in the MAB and
        :math:`\theta_k` is the parameter of the Bernoulli likelihood for arm
        `k`. The sampler begins by selecting an arm with probability
        proportional to its payoff probability under the initial Beta prior.
        After pulling the sampled arm and receiving a reward, `r`, the sampler
        computes the posterior over the model parameters (arm payoffs) via
        Bayes' rule, and then samples a new action in proportion to its payoff
        probability under this posterior. This process (i.e., sample action
        from posterior, take action and receive reward, compute updated
        posterior) is repeated until the number of trials is exhausted.

        Note that due to the conjugacy between the Beta prior and Bernoulli
        likelihood the posterior for each arm will also be Beta-distributed and
        can computed and sampled from efficiently:

        .. math::

            \theta_k \mid r \sim \text{Beta}(\alpha_k + r, \beta_k + 1 - r)

        References
        ----------
        .. [1] Thompson, W. (1933). On the likelihood that one unknown
           probability exceeds another in view of the evidence of two samples.
           *Biometrika, 25(3/4)*, 285-294.
        .. [2] Chapelle, O., & Li, L. (2011). An empirical evaluation of
           Thompson sampling. *Advances in Neural Information Processing
           Systems, 24*, 2249-2257.

        Parameters
        ----------
        alpha : float or list of length `K`
            Parameter for the Beta prior on arm payouts. If a float, this value
            will be used in the prior for all of the `K` arms.
        beta : float or list of length `K`
            Parameter for the Beta prior on arm payouts. If a float, this value
            will be used in the prior for all of the `K` arms.
        """
        super().__init__()
        self.alphas, self.betas = [], []
        self.alpha, self.beta = alpha, beta
        self.is_initialized = False

    @property