(self)
| 126 | return x |
| 127 | |
| 128 | def _reset_parameters_datapt(self): |
| 129 | # not scaling the initial embeddngs. |
| 130 | for emb in [self.data_embed, self.position_embed]: |
| 131 | std = 1 / math.sqrt(self.hparams.embed_dim) |
| 132 | nn.init.trunc_normal_(emb.weight, std=std, a=-3 * std, b=3 * std) |
| 133 | |
| 134 | # transformer block weights |
| 135 | self.blocks.apply(lambda m: _init_by_depth(m, self.hparams.num_layers)) |
| 136 | self.head.apply(lambda m: _init_by_depth(m, 1 / 2)) |
| 137 | |
| 138 | def _slice(self, x): |
| 139 | start_indices = np.arange( |
no test coverage detected