| 46 | |
| 47 | |
| 48 | def train(data_file, use_gpu, num_epoch=10, batch_size=100): |
| 49 | print('Start intialization............') |
| 50 | lr = 0.0005 # Learning rate |
| 51 | weight_decay = 0.0002 |
| 52 | hdim = 1000 |
| 53 | vdim = 784 |
| 54 | tweight = tensor.Tensor((vdim, hdim)) |
| 55 | tweight.gaussian(0.0, 0.1) |
| 56 | tvbias = tensor.from_numpy(np.zeros(vdim, dtype=np.float32)) |
| 57 | thbias = tensor.from_numpy(np.zeros(hdim, dtype=np.float32)) |
| 58 | sgd = opt.SGD(lr=lr, momentum=0.9, weight_decay=weight_decay) |
| 59 | |
| 60 | print('Loading data ..................') |
| 61 | train_x, valid_x = load_train_data(data_file) |
| 62 | |
| 63 | if use_gpu: |
| 64 | dev = device.create_cuda_gpu() |
| 65 | else: |
| 66 | dev = device.get_default_device() |
| 67 | |
| 68 | for t in [tweight, tvbias, thbias]: |
| 69 | t.to_device(dev) |
| 70 | |
| 71 | num_train_batch = train_x.shape[0] // batch_size |
| 72 | print("num_train_batch = %d " % (num_train_batch)) |
| 73 | for epoch in range(num_epoch): |
| 74 | trainerrorsum = 0.0 |
| 75 | print('Epoch %d' % epoch) |
| 76 | for b in range(num_train_batch): |
| 77 | # positive phase |
| 78 | tdata = tensor.from_numpy( |
| 79 | train_x[(b * batch_size):((b + 1) * batch_size), :]) |
| 80 | tdata.to_device(dev) |
| 81 | tposhidprob = tensor.mult(tdata, tweight) |
| 82 | tposhidprob = tposhidprob + thbias |
| 83 | tposhidprob = tensor.sigmoid(tposhidprob) |
| 84 | tposhidrandom = tensor.Tensor(tposhidprob.shape, dev) |
| 85 | tposhidrandom.uniform(0.0, 1.0) |
| 86 | tposhidsample = tensor.gt(tposhidprob, tposhidrandom) |
| 87 | |
| 88 | # negative phase |
| 89 | tnegdata = tensor.mult(tposhidsample, tweight.T()) |
| 90 | tnegdata = tnegdata + tvbias |
| 91 | tnegdata = tensor.sigmoid(tnegdata) |
| 92 | |
| 93 | tneghidprob = tensor.mult(tnegdata, tweight) |
| 94 | tneghidprob = tneghidprob + thbias |
| 95 | tneghidprob = tensor.sigmoid(tneghidprob) |
| 96 | error = tensor.sum(tensor.square((tdata - tnegdata))) |
| 97 | trainerrorsum = error + trainerrorsum |
| 98 | |
| 99 | tgweight = tensor.mult(tnegdata.T(), tneghidprob) \ |
| 100 | - tensor.mult(tdata.T(), tposhidprob) |
| 101 | tgvbias = tensor.sum(tnegdata, 0) - tensor.sum(tdata, 0) |
| 102 | tghbias = tensor.sum(tneghidprob, 0) - tensor.sum(tposhidprob, 0) |
| 103 | |
| 104 | sgd.apply('w', tweight, tgweight) |
| 105 | sgd.apply('vb', tvbias, tgvbias) |