| 69 | return X.dot(self.W) + self.b |
| 70 | |
| 71 | def sgd(self, X, Y, learning_rate=0.01, momentum=0.9): |
| 72 | # make sure X is N x D |
| 73 | assert(len(X.shape) == 2) |
| 74 | |
| 75 | # the loss values are 2-D |
| 76 | # normally we would divide by N only |
| 77 | # but now we divide by N x K |
| 78 | num_values = np.prod(Y.shape) |
| 79 | |
| 80 | # do one step of gradient descent |
| 81 | # we multiply by 2 to get the exact gradient |
| 82 | # (not adjusting the learning rate) |
| 83 | # i.e. d/dx (x^2) --> 2x |
| 84 | Yhat = self.predict(X) |
| 85 | gW = 2 * X.T.dot(Yhat - Y) / num_values |
| 86 | gb = 2 * (Yhat - Y).sum(axis=0) / num_values |
| 87 | |
| 88 | # update momentum terms |
| 89 | self.vW = momentum * self.vW - learning_rate * gW |
| 90 | self.vb = momentum * self.vb - learning_rate * gb |
| 91 | |
| 92 | # update params |
| 93 | self.W += self.vW |
| 94 | self.b += self.vb |
| 95 | |
| 96 | mse = np.mean((Yhat - Y)**2) |
| 97 | self.losses.append(mse) |
| 98 | |
| 99 | def load_weights(self, filepath): |
| 100 | npz = np.load(filepath) |