| 13 | |
| 14 | |
| 15 | class MLP(Network): |
| 16 | def __init__(self, inpt_shape, hiddens, activation=dy.rectify, layer_norm=False, pc=None): |
| 17 | super().__init__(pc) |
| 18 | if len(inpt_shape) != 1: |
| 19 | raise ValueError("inpt_shape must be 1 dimension for MLP.") |
| 20 | self.specified_activation = hasattr(activation, "__len__") |
| 21 | self.activation = activation |
| 22 | self.layer_norm = layer_norm |
| 23 | units = [inpt_shape[0]] + hiddens |
| 24 | self.Ws, self.bs = [], [] |
| 25 | if layer_norm: |
| 26 | self.ln_gs, self.ln_bs = [], [] |
| 27 | for i in range(len(units) - 1): |
| 28 | self.Ws.append(self.pc.add_parameters((units[i + 1], units[i]))) |
| 29 | self.bs.append(self.pc.add_parameters(units[i + 1])) |
| 30 | if layer_norm: |
| 31 | self.ln_gs.append(self.pc.add_parameters(units[i + 1])) |
| 32 | self.ln_bs.append(self.pc.add_parameters(units[i + 1])) |
| 33 | self.n_layers = len(self.Ws) |
| 34 | |
| 35 | def __call__(self, obs, batched=False): |
| 36 | out = obs if isinstance(obs, dy.Expression) else dy.inputTensor(obs, batched=batched) |
| 37 | |
| 38 | for i in range(self.n_layers): |
| 39 | b, W = dy.parameter(self.bs[i]), dy.parameter(self.Ws[i]) |
| 40 | out = dy.affine_transform([b, W, out]) |
| 41 | if self.layer_norm and i != self.n_layers - 1: |
| 42 | out = dy.layer_norm(out, self.ln_gs[i], self.ln_bs[i]) |
| 43 | if self.specified_activation: |
| 44 | if self.activation[i] is not None: |
| 45 | out = self.activation[i](out) |
| 46 | else: |
| 47 | out = self.activation(out) |
| 48 | return out |
| 49 | |
| 50 | |
| 51 | class Header(Network): |