Reference: attention is all you need
| 211 | return region_embedding |
| 212 | |
| 213 | class PositionEmbedding(torch.nn.Module): |
| 214 | ''' Reference: attention is all you need ''' |
| 215 | |
| 216 | def __init__(self, seq_max_len, embedding_dim, padding_idx): |
| 217 | super(PositionEmbedding, self).__init__() |
| 218 | |
| 219 | self.position_enc = nn.Embedding.from_pretrained( |
| 220 | self.get_sinusoid_encoding_table(seq_max_len + 1, |
| 221 | embedding_dim, |
| 222 | padding_idx=padding_idx), |
| 223 | freeze=True) |
| 224 | |
| 225 | def forward(self, src_pos): |
| 226 | return self.position_enc(src_pos) |
| 227 | |
| 228 | @staticmethod |
| 229 | def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None): |
| 230 | |
| 231 | def cal_angle(position, hid_idx): |
| 232 | return position / np.power(10000, 2 * (hid_idx // 2) / d_hid) |
| 233 | |
| 234 | def get_posi_angle_vec(position): |
| 235 | return [cal_angle(position, hid_j) for hid_j in range(d_hid)] |
| 236 | |
| 237 | sinusoid_table = np.array( |
| 238 | [get_posi_angle_vec(pos_i) for pos_i in range(n_position)]) |
| 239 | |
| 240 | sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i |
| 241 | sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 |
| 242 | |
| 243 | if padding_idx is not None: |
| 244 | # zero vector for padding dimension |
| 245 | sinusoid_table[padding_idx] = 0. |
| 246 | |
| 247 | return torch.FloatTensor(sinusoid_table) |