| 102 | # value_net (ValueNetwork) should be a global passed to every worker |
| 103 | # returns_list (List) should be a global passed to every worker |
| 104 | class Worker: |
| 105 | def __init__( |
| 106 | self, |
| 107 | name, |
| 108 | env, |
| 109 | policy_net, |
| 110 | value_net, |
| 111 | global_counter, |
| 112 | returns_list, |
| 113 | discount_factor=0.99, |
| 114 | max_global_steps=None): |
| 115 | |
| 116 | self.name = name |
| 117 | self.env = env |
| 118 | self.global_policy_net = policy_net |
| 119 | self.global_value_net = value_net |
| 120 | self.global_counter = global_counter |
| 121 | self.discount_factor = discount_factor |
| 122 | self.max_global_steps = max_global_steps |
| 123 | self.global_step = tf.train.get_global_step() |
| 124 | self.img_transformer = ImageTransformer() |
| 125 | |
| 126 | # Create local policy and value networks that belong only to this worker |
| 127 | with tf.variable_scope(name): |
| 128 | # self.policy_net = PolicyNetwork(num_outputs=policy_net.num_outputs) |
| 129 | # self.value_net = ValueNetwork() |
| 130 | self.policy_net, self.value_net = create_networks(policy_net.num_outputs) |
| 131 | |
| 132 | # We will use this op to copy the global network weights |
| 133 | # back to the local policy and value networks |
| 134 | self.copy_params_op = get_copy_params_op( |
| 135 | tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="global"), |
| 136 | tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name+'/')) |
| 137 | |
| 138 | # These will take the gradients from the local networks |
| 139 | # and use those gradients to update the global network |
| 140 | self.vnet_train_op = make_train_op(self.value_net, self.global_value_net) |
| 141 | self.pnet_train_op = make_train_op(self.policy_net, self.global_policy_net) |
| 142 | |
| 143 | self.state = None # Keep track of the current state |
| 144 | self.total_reward = 0. # After each episode print the total (sum of) reward |
| 145 | self.returns_list = returns_list # Global returns list to plot later |
| 146 | |
| 147 | def run(self, sess, coord, t_max): |
| 148 | with sess.as_default(), sess.graph.as_default(): |
| 149 | # Assign the initial state |
| 150 | self.state = repeat_frame(self.img_transformer.transform(self.env.reset())) |
| 151 | |
| 152 | try: |
| 153 | while not coord.should_stop(): |
| 154 | # Copy weights from global networks to local networks |
| 155 | sess.run(self.copy_params_op) |
| 156 | |
| 157 | # Collect some experience |
| 158 | steps, global_step = self.run_n_steps(t_max, sess) |
| 159 | |
| 160 | # Stop once the max number of global steps has been reached |
| 161 | if self.max_global_steps is not None and global_step >= self.max_global_steps: |