| 573 | return enc_output |
| 574 | |
| 575 | class PositionalEncoding(nn.Module): |
| 576 | |
| 577 | def __init__(self, d_hid, n_position=20): |
| 578 | super(PositionalEncoding, self).__init__() |
| 579 | self.register_buffer('pos_table', self._get_sinusoid_encoding_table(n_position, d_hid)) |
| 580 | |
| 581 | def _get_sinusoid_encoding_table(self, n_position, d_hid): |
| 582 | def get_position_angle_vec(position): |
| 583 | return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] |
| 584 | |
| 585 | sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) |
| 586 | sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) |
| 587 | sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) |
| 588 | |
| 589 | return torch.FloatTensor(sinusoid_table).unsqueeze(0) |
| 590 | |
| 591 | def forward(self, x): |
| 592 | return self.pos_table[:, :x].clone().detach() |
| 593 | |
| 594 | class Model(nn.Module): |
| 595 | def __init__(self, args, meta_shape=None, cross_shape=None, growth_rate=12, num_init_features=12, bn_size=4, drop_rate=0.2, nb_flows=1): |