Class ReplayBuffer

core/replay_buffer.py:8–173

Reference : DISTRIBUTED PRIORITIZED EXPERIENCE REPLAY Algo. 1 and Algo. 2 in Page-3 of (https://arxiv.org/pdf/1803.00933.pdf

@ray.remote
class ReplayBuffer(object):
    """Reference : DISTRIBUTED PRIORITIZED EXPERIENCE REPLAY
    Algo. 1 and Algo. 2 in Page-3 of (https://arxiv.org/pdf/1803.00933.pdf
    """
    def __init__(self, config=None):
        self.config = config
        self.batch_size = config.batch_size
        self.keep_ratio = 1

        self.model_index = 0
        self.model_update_interval = 10

        self.buffer = []
        self.priorities = []
        self.game_look_up = []

        self._eps_collected = 0
        self.base_idx = 0
        self._alpha = config.priority_prob_alpha
        self.transition_top = int(config.transition_num * 10 ** 6)
        self.clear_time = 0

    def save_pools(self, pools, gap_step):
        # save a list of game histories
        for (game, priorities) in pools:
            # Only append end game
            # if end_tag:
            if len(game) > 0:
                self.save_game(game, True, gap_step, priorities)

    def save_game(self, game, end_tag, gap_steps, priorities=None):
        """Save a game history block
        Parameters
        ----------
        game: Any
            a game history block
        end_tag: bool
            True -> the game is finished. (always True)
        gap_steps: int
            if the game is not finished, we only save the transitions that can be computed
        priorities: list
            the priorities corresponding to the transitions in the game history
        """
        if self.get_total_len() >= self.config.total_transitions:
            return

        if end_tag:
            self._eps_collected += 1
            valid_len = len(game)
        else:
            valid_len = len(game) - gap_steps

        if priorities is None:
            max_prio = self.priorities.max() if self.buffer else 1
            self.priorities = np.concatenate((self.priorities, [max_prio for _ in range(valid_len)] + [0. for _ in range(valid_len, len(game))]))
        else:
            assert len(game) == len(priorities), " priorities should be of same length as the game steps"
            priorities = priorities.copy().reshape(-1)