(self, d_model, n_head, window_size=None, drop_path_rate=0.0)
| 258 | Attention block. |
| 259 | """ |
| 260 | def __init__(self, d_model, n_head, window_size=None, drop_path_rate=0.0): |
| 261 | super().__init__() |
| 262 | self.attn = MultiHeadAttention(d_model, d_model, d_model, d_model, n_head) |
| 263 | self.layernorm1 = LayerNorm(d_model) |
| 264 | self.mlp = nn.Sequential(OrderedDict([ |
| 265 | ("c_fc", nn.Linear(d_model, d_model * 4, bias=False)), |
| 266 | ("silu", nn.SiLU(inplace=True)), |
| 267 | ("c_proj", nn.Linear(d_model * 4, d_model, bias=False)) |
| 268 | ])) |
| 269 | self.layernorm2 = LayerNorm(d_model) |
| 270 | self.window_size = window_size |
| 271 | |
| 272 | def attention(self, x, index): |
| 273 | attn_mask = None |
nothing calls this directly
no test coverage detected