| 10 | import warnings |
| 11 | |
| 12 | def create_PositionalEncoding(input_dim, max_seq_len=2000): |
| 13 | position_encoding = np.array([ |
| 14 | [pos / np.power(10000, 2.0 * (j // 2) / input_dim) for j in range(input_dim)] |
| 15 | for pos in range(max_seq_len)]) |
| 16 | |
| 17 | position_encoding[:, 0::2] = np.sin(position_encoding[:, 0::2]) |
| 18 | position_encoding[:, 1::2] = np.cos(position_encoding[:, 1::2]) |
| 19 | |
| 20 | position_encoding = torch.from_numpy(position_encoding.astype(np.float32)) |
| 21 | position_encoding = nn.Parameter(position_encoding, requires_grad=False) |
| 22 | |
| 23 | return position_encoding |
| 24 | |
| 25 | def _get_activation_fn(activation: str='relu', module: bool=False): |
| 26 | """ Returns the activation function corresponding to `activation` """ |