| 207 | |
| 208 | @staticmethod |
| 209 | def _get_sinusoid_encoding_table(n_position, d_model): |
| 210 | def cal_angle(position, hid_idx): |
| 211 | return position / np.power(10000, 2 * (hid_idx // 2) / d_model) |
| 212 | |
| 213 | def get_posi_angle_vec(position): |
| 214 | return [cal_angle(position, hid_j) for hid_j in range(d_model)] |
| 215 | |
| 216 | sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)], np.float32) |
| 217 | sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # Even bits use sine functions |
| 218 | sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # Cosine function for odd digits |
| 219 | return tensor.Tensor(data=sinusoid_table, requires_grad=False) |
| 220 | |
| 221 | |
| 222 | class TransformerDecoderLayer(layer.Layer): |