Construct an PositionalEncoding object.
(self,
d_model: int,
dropout_rate: float,
max_len: int = 5000,
reverse: bool = False)
| 36 | """ |
| 37 | |
| 38 | def __init__(self, |
| 39 | d_model: int, |
| 40 | dropout_rate: float, |
| 41 | max_len: int = 5000, |
| 42 | reverse: bool = False): |
| 43 | """Construct an PositionalEncoding object.""" |
| 44 | super().__init__() |
| 45 | self.d_model = d_model |
| 46 | self.xscale = math.sqrt(self.d_model) |
| 47 | self.dropout = torch.nn.Dropout(p=dropout_rate) |
| 48 | self.max_len = max_len |
| 49 | |
| 50 | pe = torch.zeros(self.max_len, self.d_model) |
| 51 | position = torch.arange(0, self.max_len, |
| 52 | dtype=torch.float32).unsqueeze(1) |
| 53 | div_term = torch.exp( |
| 54 | torch.arange(0, self.d_model, 2, dtype=torch.float32) * |
| 55 | -(math.log(10000.0) / self.d_model)) |
| 56 | pe[:, 0::2] = torch.sin(position * div_term) |
| 57 | pe[:, 1::2] = torch.cos(position * div_term) |
| 58 | pe = pe.unsqueeze(0) |
| 59 | self.register_buffer("pe", pe) |
| 60 | |
| 61 | def forward(self, |
| 62 | x: torch.Tensor, |