(self)
| 116 | return self.char_to_idx[self.text[pos]] |
| 117 | |
| 118 | def TrainModel(self): |
| 119 | log.debug("Training model") |
| 120 | |
| 121 | workspace.RunNetOnce(self.model.param_init_net) |
| 122 | |
| 123 | # As though we predict the same probability for each character |
| 124 | smooth_loss = -np.log(1.0 / self.D) * self.seq_length |
| 125 | last_n_iter = 0 |
| 126 | last_n_loss = 0.0 |
| 127 | num_iter = 0 |
| 128 | N = len(self.text) |
| 129 | |
| 130 | # We split text into batch_size pieces. Each piece will be used only |
| 131 | # by a corresponding batch during the training process |
| 132 | text_block_positions = np.zeros(self.batch_size, dtype=np.int32) |
| 133 | text_block_size = N // self.batch_size |
| 134 | text_block_starts = list(range(0, N, text_block_size)) |
| 135 | text_block_sizes = [text_block_size] * self.batch_size |
| 136 | text_block_sizes[self.batch_size - 1] += N % self.batch_size |
| 137 | assert sum(text_block_sizes) == N |
| 138 | |
| 139 | # Writing to output states which will be copied to input |
| 140 | # states within the loop below |
| 141 | workspace.FeedBlob(self.hidden_output, np.zeros( |
| 142 | [1, self.batch_size, self.hidden_size], dtype=np.float32 |
| 143 | )) |
| 144 | workspace.FeedBlob(self.cell_state, np.zeros( |
| 145 | [1, self.batch_size, self.hidden_size], dtype=np.float32 |
| 146 | )) |
| 147 | workspace.CreateNet(self.prepare_state) |
| 148 | |
| 149 | # We iterate over text in a loop many times. Each time we peak |
| 150 | # seq_length segment and feed it to LSTM as a sequence |
| 151 | last_time = datetime.now() |
| 152 | progress = 0 |
| 153 | while True: |
| 154 | workspace.FeedBlob( |
| 155 | "seq_lengths", |
| 156 | np.array([self.seq_length] * self.batch_size, |
| 157 | dtype=np.int32) |
| 158 | ) |
| 159 | workspace.RunNet(self.prepare_state.Name()) |
| 160 | |
| 161 | input = np.zeros( |
| 162 | [self.seq_length, self.batch_size, self.D] |
| 163 | ).astype(np.float32) |
| 164 | target = np.zeros( |
| 165 | [self.seq_length * self.batch_size] |
| 166 | ).astype(np.int32) |
| 167 | |
| 168 | for e in range(self.batch_size): |
| 169 | for i in range(self.seq_length): |
| 170 | pos = text_block_starts[e] + text_block_positions[e] |
| 171 | input[i][e][self._idx_at_pos(pos)] = 1 |
| 172 | target[i * self.batch_size + e] =\ |
| 173 | self._idx_at_pos((pos + 1) % N) |
| 174 | text_block_positions[e] = ( |
| 175 | text_block_positions[e] + 1) % text_block_sizes[e] |
no test coverage detected