Sinusoidal position encoding table. Args: n_position (int): the length of the input sequence d_hid (int): the dimension of the hidden state
(n_position, d_hid, padding_idx=None)
| 17 | |
| 18 | |
| 19 | def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None): |
| 20 | """ |
| 21 | Sinusoidal position encoding table. |
| 22 | Args: |
| 23 | n_position (int): the length of the input sequence |
| 24 | d_hid (int): the dimension of the hidden state |
| 25 | """ |
| 26 | def cal_angle(position, hid_idx): |
| 27 | return position / (10000 ** (2 * (hid_idx // 2) / d_hid)) |
| 28 | |
| 29 | def get_posi_angle_vec(position): |
| 30 | return [cal_angle(position, hid_j) for hid_j in range(d_hid)] |
| 31 | |
| 32 | sinusoid_table = torch.Tensor([get_posi_angle_vec(pos_i) for pos_i in range(n_position)]) |
| 33 | sinusoid_table[:, 0::2] = torch.sin(sinusoid_table[:, 0::2]) # dim 2i |
| 34 | sinusoid_table[:, 1::2] = torch.cos(sinusoid_table[:, 1::2]) # dim 2i+1 |
| 35 | if padding_idx is not None: sinusoid_table[padding_idx] = 0. |
| 36 | return sinusoid_table |
| 37 | |
| 38 | |
| 39 | class Attention(nn.Module): |
no test coverage detected