| 2 | |
| 3 | |
| 4 | class ValueIteration: |
| 5 | def __init__(self, env): |
| 6 | self.env = env |
| 7 | # 2-d list for the value function |
| 8 | self.value_table = [[0.0] * env.width for _ in range(env.height)] |
| 9 | self.discount_factor = 0.9 |
| 10 | |
| 11 | # get next value function table from the current value function table |
| 12 | def value_iteration(self): |
| 13 | next_value_table = [[0.0] * self.env.width |
| 14 | for _ in range(self.env.height)] |
| 15 | for state in self.env.get_all_states(): |
| 16 | if state == [2, 2]: |
| 17 | next_value_table[state[0]][state[1]] = 0.0 |
| 18 | continue |
| 19 | value_list = [] |
| 20 | |
| 21 | for action in self.env.possible_actions: |
| 22 | next_state = self.env.state_after_action(state, action) |
| 23 | reward = self.env.get_reward(state, action) |
| 24 | next_value = self.get_value(next_state) |
| 25 | value_list.append((reward + self.discount_factor * next_value)) |
| 26 | # return the maximum value(it is the optimality equation!!) |
| 27 | next_value_table[state[0]][state[1]] = round(max(value_list), 2) |
| 28 | self.value_table = next_value_table |
| 29 | |
| 30 | # get action according to the current value function table |
| 31 | def get_action(self, state): |
| 32 | action_list = [] |
| 33 | max_value = -99999 |
| 34 | |
| 35 | if state == [2, 2]: |
| 36 | return [] |
| 37 | |
| 38 | # calculating q values for the all actions and |
| 39 | # append the action to action list which has maximum q value |
| 40 | for action in self.env.possible_actions: |
| 41 | |
| 42 | next_state = self.env.state_after_action(state, action) |
| 43 | reward = self.env.get_reward(state, action) |
| 44 | next_value = self.get_value(next_state) |
| 45 | value = (reward + self.discount_factor * next_value) |
| 46 | |
| 47 | if value > max_value: |
| 48 | action_list.clear() |
| 49 | action_list.append(action) |
| 50 | max_value = value |
| 51 | elif value == max_value: |
| 52 | action_list.append(action) |
| 53 | |
| 54 | return action_list |
| 55 | |
| 56 | def get_value(self, state): |
| 57 | return round(self.value_table[state[0]][state[1]], 2) |
| 58 | |
| 59 | if __name__ == "__main__": |
| 60 | env = Env() |