()
| 16 | |
| 17 | |
| 18 | def main(): |
| 19 | max_iter = 20 # make it 30 for sigmoid |
| 20 | print_period = 10 |
| 21 | |
| 22 | Xtrain, Xtest, Ytrain, Ytest = get_normalized_data() |
| 23 | lr = 0.00004 |
| 24 | reg = 0.01 |
| 25 | |
| 26 | Ytrain_ind = y2indicator(Ytrain) |
| 27 | Ytest_ind = y2indicator(Ytest) |
| 28 | |
| 29 | N, D = Xtrain.shape |
| 30 | batch_sz = 500 |
| 31 | n_batches = N // batch_sz |
| 32 | |
| 33 | M = 300 |
| 34 | K = 10 |
| 35 | W1 = np.random.randn(D, M) / np.sqrt(D) |
| 36 | b1 = np.zeros(M) |
| 37 | W2 = np.random.randn(M, K) / np.sqrt(M) |
| 38 | b2 = np.zeros(K) |
| 39 | |
| 40 | # 1. const |
| 41 | # cost = -16 |
| 42 | LL_batch = [] |
| 43 | CR_batch = [] |
| 44 | for i in range(max_iter): |
| 45 | for j in range(n_batches): |
| 46 | Xbatch = Xtrain[j*batch_sz:(j*batch_sz + batch_sz),] |
| 47 | Ybatch = Ytrain_ind[j*batch_sz:(j*batch_sz + batch_sz),] |
| 48 | pYbatch, Z = forward(Xbatch, W1, b1, W2, b2) |
| 49 | # print "first batch cost:", cost(pYbatch, Ybatch) |
| 50 | |
| 51 | # gradients |
| 52 | gW2 = derivative_w2(Z, Ybatch, pYbatch) + reg*W2 |
| 53 | gb2 = derivative_b2(Ybatch, pYbatch) + reg*b2 |
| 54 | gW1 = derivative_w1(Xbatch, Z, Ybatch, pYbatch, W2) + reg*W1 |
| 55 | gb1 = derivative_b1(Z, Ybatch, pYbatch, W2) + reg*b1 |
| 56 | |
| 57 | # updates |
| 58 | W2 -= lr*gW2 |
| 59 | b2 -= lr*gb2 |
| 60 | W1 -= lr*gW1 |
| 61 | b1 -= lr*gb1 |
| 62 | |
| 63 | if j % print_period == 0: |
| 64 | # calculate just for LL |
| 65 | pY, _ = forward(Xtest, W1, b1, W2, b2) |
| 66 | # print "pY:", pY |
| 67 | ll = cost(pY, Ytest_ind) |
| 68 | LL_batch.append(ll) |
| 69 | print("Cost at iteration i=%d, j=%d: %.6f" % (i, j, ll)) |
| 70 | |
| 71 | err = error_rate(pY, Ytest) |
| 72 | CR_batch.append(err) |
| 73 | print("Error rate:", err) |
| 74 | |
| 75 | pY, _ = forward(Xtest, W1, b1, W2, b2) |
no test coverage detected