| 579 | self.register_buffer('pos_table', self._get_sinusoid_encoding_table(n_position, d_hid)) |
| 580 | |
| 581 | def _get_sinusoid_encoding_table(self, n_position, d_hid): |
| 582 | def get_position_angle_vec(position): |
| 583 | return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] |
| 584 | |
| 585 | sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) |
| 586 | sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) |
| 587 | sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) |
| 588 | |
| 589 | return torch.FloatTensor(sinusoid_table).unsqueeze(0) |
| 590 | |
| 591 | def forward(self, x): |
| 592 | return self.pos_table[:, :x].clone().detach() |