(self)
| 54 | len(self.vocab), len(self.text))) |
| 55 | |
| 56 | def CreateModel(self): |
| 57 | log.debug("Start training") |
| 58 | model = model_helper.ModelHelper(name="char_rnn") |
| 59 | |
| 60 | input_blob, seq_lengths, hidden_init, cell_init, target = \ |
| 61 | model.net.AddExternalInputs( |
| 62 | 'input_blob', |
| 63 | 'seq_lengths', |
| 64 | 'hidden_init', |
| 65 | 'cell_init', |
| 66 | 'target', |
| 67 | ) |
| 68 | |
| 69 | hidden_output_all, self.hidden_output, _, self.cell_state = LSTM( |
| 70 | model, input_blob, seq_lengths, (hidden_init, cell_init), |
| 71 | self.D, self.hidden_size, scope="LSTM") |
| 72 | output = brew.fc( |
| 73 | model, |
| 74 | hidden_output_all, |
| 75 | None, |
| 76 | dim_in=self.hidden_size, |
| 77 | dim_out=self.D, |
| 78 | axis=2 |
| 79 | ) |
| 80 | |
| 81 | # axis is 2 as first two are T (time) and N (batch size). |
| 82 | # We treat them as one big batch of size T * N |
| 83 | softmax = model.net.Softmax(output, 'softmax', axis=2) |
| 84 | |
| 85 | softmax_reshaped, _ = model.net.Reshape( |
| 86 | softmax, ['softmax_reshaped', '_'], shape=[-1, self.D]) |
| 87 | |
| 88 | # Create a copy of the current net. We will use it on the forward |
| 89 | # pass where we don't need loss and backward operators |
| 90 | self.forward_net = core.Net(model.net.Proto()) |
| 91 | |
| 92 | xent = model.net.LabelCrossEntropy([softmax_reshaped, target], 'xent') |
| 93 | # Loss is average both across batch and through time |
| 94 | # Thats why the learning rate below is multiplied by self.seq_length |
| 95 | loss = model.net.AveragedLoss(xent, 'loss') |
| 96 | model.AddGradientOperators([loss]) |
| 97 | |
| 98 | # use build_sdg function to build an optimizer |
| 99 | build_sgd( |
| 100 | model, |
| 101 | base_learning_rate=0.1 * self.seq_length, |
| 102 | policy="step", |
| 103 | stepsize=1, |
| 104 | gamma=0.9999 |
| 105 | ) |
| 106 | |
| 107 | self.model = model |
| 108 | self.predictions = softmax |
| 109 | self.loss = loss |
| 110 | |
| 111 | self.prepare_state = core.Net("prepare_state") |
| 112 | self.prepare_state.Copy(self.hidden_output, hidden_init) |
| 113 | self.prepare_state.Copy(self.cell_state, cell_init) |
no test coverage detected