prepare the context of a batch reward_value_context: the context of reanalyzed value targets policy_re_context: the context of reanalyzed policy targets policy_non_re_context: the context of non-reanalyzed policy targets inputs_batch:
(self, batch_context, ratio, weights=None)
| 159 | return policy_re_context |
| 160 | |
| 161 | def make_batch(self, batch_context, ratio, weights=None): |
| 162 | """prepare the context of a batch |
| 163 | reward_value_context: the context of reanalyzed value targets |
| 164 | policy_re_context: the context of reanalyzed policy targets |
| 165 | policy_non_re_context: the context of non-reanalyzed policy targets |
| 166 | inputs_batch: the inputs of batch |
| 167 | weights: the target model weights |
| 168 | Parameters |
| 169 | ---------- |
| 170 | batch_context: Any |
| 171 | batch context from replay buffer |
| 172 | ratio: float |
| 173 | ratio of reanalyzed policy (value is 100% reanalyzed) |
| 174 | weights: Any |
| 175 | the target model weights |
| 176 | """ |
| 177 | # obtain the batch context from replay buffer |
| 178 | game_lst, game_pos_lst, indices_lst, weights_lst, make_time_lst = batch_context |
| 179 | batch_size = len(indices_lst) |
| 180 | obs_lst, action_lst, mask_lst = [], [], [] |
| 181 | # prepare the inputs of a batch |
| 182 | for i in range(batch_size): |
| 183 | game = game_lst[i] |
| 184 | game_pos = game_pos_lst[i] |
| 185 | |
| 186 | _actions = game.actions[game_pos:game_pos + self.config.num_unroll_steps].tolist() |
| 187 | # add mask for invalid actions (out of trajectory) |
| 188 | _mask = [1. for i in range(len(_actions))] |
| 189 | _mask += [0. for _ in range(self.config.num_unroll_steps - len(_mask))] |
| 190 | |
| 191 | _actions += [np.random.randint(0, game.action_space_size) for _ in range(self.config.num_unroll_steps - len(_actions))] |
| 192 | |
| 193 | # obtain the input observations |
| 194 | obs_lst.append(game_lst[i].obs(game_pos_lst[i], extra_len=self.config.num_unroll_steps, padding=True)) |
| 195 | action_lst.append(_actions) |
| 196 | mask_lst.append(_mask) |
| 197 | |
| 198 | re_num = int(batch_size * ratio) |
| 199 | # formalize the input observations |
| 200 | obs_lst = prepare_observation_lst(obs_lst) |
| 201 | |
| 202 | # formalize the inputs of a batch |
| 203 | inputs_batch = [obs_lst, action_lst, mask_lst, indices_lst, weights_lst, make_time_lst] |
| 204 | for i in range(len(inputs_batch)): |
| 205 | inputs_batch[i] = np.asarray(inputs_batch[i]) |
| 206 | |
| 207 | total_transitions = ray.get(self.replay_buffer.get_total_len.remote()) |
| 208 | |
| 209 | # obtain the context of value targets |
| 210 | reward_value_context = self._prepare_reward_value_context(indices_lst, game_lst, game_pos_lst, total_transitions) |
| 211 | |
| 212 | # 0:re_num -> reanalyzed policy, re_num:end -> non reanalyzed policy |
| 213 | # reanalyzed policy |
| 214 | if re_num > 0: |
| 215 | # obtain the context of reanalyzed policy targets |
| 216 | policy_re_context = self._prepare_policy_re_context(indices_lst[:re_num], game_lst[:re_num], game_pos_lst[:re_num]) |
| 217 | else: |
| 218 | policy_re_context = None |
no test coverage detected