| 29 | |
| 30 | |
| 31 | class Agent: |
| 32 | def __init__(self, eps=0.1, alpha=0.5): |
| 33 | self.eps = eps # probability of choosing random action instead of greedy |
| 34 | self.alpha = alpha # learning rate |
| 35 | self.verbose = False |
| 36 | self.state_history = [] |
| 37 | |
| 38 | def setV(self, V): |
| 39 | self.V = V |
| 40 | |
| 41 | def set_symbol(self, sym): |
| 42 | self.sym = sym |
| 43 | |
| 44 | def set_verbose(self, v): |
| 45 | # if true, will print values for each position on the board |
| 46 | self.verbose = v |
| 47 | |
| 48 | def reset_history(self): |
| 49 | self.state_history = [] |
| 50 | |
| 51 | def take_action(self, env): |
| 52 | # choose an action based on epsilon-greedy strategy |
| 53 | r = np.random.rand() |
| 54 | best_state = None |
| 55 | if r < self.eps: |
| 56 | # take a random action |
| 57 | if self.verbose: |
| 58 | print("Taking a random action") |
| 59 | |
| 60 | possible_moves = [] |
| 61 | for i in range(LENGTH): |
| 62 | for j in range(LENGTH): |
| 63 | if env.is_empty(i, j): |
| 64 | possible_moves.append((i, j)) |
| 65 | idx = np.random.choice(len(possible_moves)) |
| 66 | next_move = possible_moves[idx] |
| 67 | else: |
| 68 | # choose the best action based on current values of states |
| 69 | # loop through all possible moves, get their values |
| 70 | # keep track of the best value |
| 71 | pos2value = {} # for debugging |
| 72 | next_move = None |
| 73 | best_value = -1 |
| 74 | for i in range(LENGTH): |
| 75 | for j in range(LENGTH): |
| 76 | if env.is_empty(i, j): |
| 77 | # what is the state if we made this move? |
| 78 | env.board[i,j] = self.sym |
| 79 | state = env.get_state() |
| 80 | env.board[i,j] = 0 # don't forget to change it back! |
| 81 | pos2value[(i,j)] = self.V[state] |
| 82 | if self.V[state] > best_value: |
| 83 | best_value = self.V[state] |
| 84 | best_state = state |
| 85 | next_move = (i, j) |
| 86 | |
| 87 | # if verbose, draw the board w/ the values |
| 88 | if self.verbose: |