| 68 | self.bottom_diff_x = np.zeros(x_dim) |
| 69 | |
| 70 | class LstmNode: |
| 71 | def __init__(self, lstm_param, lstm_state): |
| 72 | # store reference to parameters and to activations |
| 73 | self.state = lstm_state |
| 74 | self.param = lstm_param |
| 75 | # non-recurrent input to node |
| 76 | self.x = None |
| 77 | # non-recurrent input concatenated with recurrent input |
| 78 | self.xc = None |
| 79 | |
| 80 | def bottom_data_is(self, x, s_prev = None, h_prev = None): |
| 81 | # if this is the first lstm node in the network |
| 82 | if s_prev == None: s_prev = np.zeros_like(self.state.s) |
| 83 | if h_prev == None: h_prev = np.zeros_like(self.state.h) |
| 84 | # save data for use in backprop |
| 85 | self.s_prev = s_prev |
| 86 | self.h_prev = h_prev |
| 87 | |
| 88 | # concatenate x(t) and h(t-1) |
| 89 | xc = np.hstack((x, h_prev)) |
| 90 | self.state.g = np.tanh(np.dot(self.param.wg, xc) + self.param.bg) |
| 91 | self.state.i = sigmoid(np.dot(self.param.wi, xc) + self.param.bi) |
| 92 | self.state.f = sigmoid(np.dot(self.param.wf, xc) + self.param.bf) |
| 93 | self.state.o = sigmoid(np.dot(self.param.wo, xc) + self.param.bo) |
| 94 | self.state.s = self.state.g * self.state.i + s_prev * self.state.f |
| 95 | self.state.h = self.state.s * self.state.o |
| 96 | self.x = x |
| 97 | self.xc = xc |
| 98 | |
| 99 | def top_diff_is(self, top_diff_h, top_diff_s): |
| 100 | # notice that top_diff_s is carried along the constant error carousel |
| 101 | ds = self.state.o * top_diff_h + top_diff_s |
| 102 | do = self.state.s * top_diff_h |
| 103 | di = self.state.g * ds |
| 104 | dg = self.state.i * ds |
| 105 | df = self.s_prev * ds |
| 106 | |
| 107 | # diffs w.r.t. vector inside sigma / tanh function |
| 108 | di_input = (1. - self.state.i) * self.state.i * di |
| 109 | df_input = (1. - self.state.f) * self.state.f * df |
| 110 | do_input = (1. - self.state.o) * self.state.o * do |
| 111 | dg_input = (1. - self.state.g ** 2) * dg |
| 112 | |
| 113 | # diffs w.r.t. inputs |
| 114 | self.param.wi_diff += np.outer(di_input, self.xc) |
| 115 | self.param.wf_diff += np.outer(df_input, self.xc) |
| 116 | self.param.wo_diff += np.outer(do_input, self.xc) |
| 117 | self.param.wg_diff += np.outer(dg_input, self.xc) |
| 118 | self.param.bi_diff += di_input |
| 119 | self.param.bf_diff += df_input |
| 120 | self.param.bo_diff += do_input |
| 121 | self.param.bg_diff += dg_input |
| 122 | |
| 123 | # compute bottom diff |
| 124 | dxc = np.zeros_like(self.xc) |
| 125 | dxc += np.dot(self.param.wi.T, di_input) |
| 126 | dxc += np.dot(self.param.wf.T, df_input) |
| 127 | dxc += np.dot(self.param.wo.T, do_input) |