| 360 | return x |
| 361 | |
| 362 | class SinusoidalPositionalEncoding(nn.Module): |
| 363 | def __init__(self, max_len, d_model): |
| 364 | super().__init__() |
| 365 | pe = torch.zeros(max_len, d_model) |
| 366 | position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) |
| 367 | div_term = torch.exp(torch.arange(0, d_model, 2, dtype=torch.float) * (-math.log(10000.0) / d_model)) |
| 368 | pe[:, 0::2] = torch.sin(position * div_term) |
| 369 | pe[:, 1::2] = torch.cos(position * div_term) |
| 370 | self.register_buffer('pe', pe) |
| 371 | |
| 372 | def forward(self, x): |
| 373 | seq_len = x.size(1) |
| 374 | return self.pe[:seq_len, :].unsqueeze(0) |
| 375 | |
| 376 | class Transformer(nn.Module): |
| 377 | def __init__(self, width, layers, heads, window_size=None, block_cls=ResAttBlock, drop_path_rate=0.0): |