| 133 | self.state.bottom_diff_h = dxc[self.param.x_dim:] |
| 134 | |
| 135 | class LstmNetwork(): |
| 136 | def __init__(self, lstm_param): |
| 137 | self.lstm_param = lstm_param |
| 138 | self.lstm_node_list = [] |
| 139 | # input sequence |
| 140 | self.x_list = [] |
| 141 | |
| 142 | def y_list_is(self, y_list, loss_layer): |
| 143 | """ |
| 144 | Updates diffs by setting target sequence |
| 145 | with corresponding loss layer. |
| 146 | Will *NOT* update parameters. To update parameters, |
| 147 | call self.lstm_param.apply_diff() |
| 148 | """ |
| 149 | assert len(y_list) == len(self.x_list) |
| 150 | idx = len(self.x_list) - 1 |
| 151 | # first node only gets diffs from label ... |
| 152 | loss = loss_layer.loss(self.lstm_node_list[idx].state.h, y_list[idx]) |
| 153 | diff_h = loss_layer.bottom_diff(self.lstm_node_list[idx].state.h, y_list[idx]) |
| 154 | # here s is not affecting loss due to h(t+1), hence we set equal to zero |
| 155 | diff_s = np.zeros(self.lstm_param.mem_cell_ct) |
| 156 | self.lstm_node_list[idx].top_diff_is(diff_h, diff_s) |
| 157 | idx -= 1 |
| 158 | |
| 159 | ### ... following nodes also get diffs from next nodes, hence we add diffs to diff_h |
| 160 | ### we also propagate error along constant error carousel using diff_s |
| 161 | while idx >= 0: |
| 162 | loss += loss_layer.loss(self.lstm_node_list[idx].state.h, y_list[idx]) |
| 163 | diff_h = loss_layer.bottom_diff(self.lstm_node_list[idx].state.h, y_list[idx]) |
| 164 | diff_h += self.lstm_node_list[idx + 1].state.bottom_diff_h |
| 165 | diff_s = self.lstm_node_list[idx + 1].state.bottom_diff_s |
| 166 | self.lstm_node_list[idx].top_diff_is(diff_h, diff_s) |
| 167 | idx -= 1 |
| 168 | |
| 169 | return loss |
| 170 | |
| 171 | def x_list_clear(self): |
| 172 | self.x_list = [] |
| 173 | |
| 174 | def x_list_add(self, x): |
| 175 | self.x_list.append(x) |
| 176 | if len(self.x_list) > len(self.lstm_node_list): |
| 177 | # need to add new lstm node, create new state mem |
| 178 | lstm_state = LstmState(self.lstm_param.mem_cell_ct, self.lstm_param.x_dim) |
| 179 | self.lstm_node_list.append(LstmNode(self.lstm_param, lstm_state)) |
| 180 | |
| 181 | # get index of most recent x input |
| 182 | idx = len(self.x_list) - 1 |
| 183 | if idx == 0: |
| 184 | # no recurrent inputs yet |
| 185 | self.lstm_node_list[idx].bottom_data_is(x) |
| 186 | else: |
| 187 | s_prev = self.lstm_node_list[idx - 1].state.s |
| 188 | h_prev = self.lstm_node_list[idx - 1].state.h |
| 189 | self.lstm_node_list[idx].bottom_data_is(x, s_prev, h_prev) |
| 190 | |