| 13 | |
| 14 | |
| 15 | class AdaBoost: |
| 16 | def __init__(self, M): |
| 17 | self.M = M |
| 18 | |
| 19 | def fit(self, X, Y): |
| 20 | self.models = [] |
| 21 | self.alphas = [] |
| 22 | |
| 23 | N, _ = X.shape |
| 24 | W = np.ones(N) / N |
| 25 | |
| 26 | for m in range(self.M): |
| 27 | tree = DecisionTreeClassifier(max_depth=1) |
| 28 | tree.fit(X, Y, sample_weight=W) |
| 29 | P = tree.predict(X) |
| 30 | |
| 31 | err = W.dot(P != Y) |
| 32 | alpha = 0.5*(np.log(1 - err) - np.log(err)) |
| 33 | |
| 34 | W = W*np.exp(-alpha*Y*P) # vectorized form |
| 35 | W = W / W.sum() # normalize so it sums to 1 |
| 36 | |
| 37 | self.models.append(tree) |
| 38 | self.alphas.append(alpha) |
| 39 | |
| 40 | def predict(self, X): |
| 41 | # NOT like SKLearn API |
| 42 | # we want accuracy and exponential loss for plotting purposes |
| 43 | N, _ = X.shape |
| 44 | FX = np.zeros(N) |
| 45 | for alpha, tree in zip(self.alphas, self.models): |
| 46 | FX += alpha*tree.predict(X) |
| 47 | return np.sign(FX), FX |
| 48 | |
| 49 | def score(self, X, Y): |
| 50 | # NOT like SKLearn API |
| 51 | # we want accuracy and exponential loss for plotting purposes |
| 52 | P, FX = self.predict(X) |
| 53 | L = np.exp(-Y*FX).mean() |
| 54 | return np.mean(P == Y), L |
| 55 | |
| 56 | |
| 57 | if __name__ == '__main__': |