| 60 | |
| 61 | |
| 62 | class Transformer(nn.Module): |
| 63 | def __init__(self, dim, depth, heads, dim_head, mlp_dim): |
| 64 | super().__init__() |
| 65 | self.norm = nn.LayerNorm(dim) |
| 66 | self.layers = nn.ModuleList([]) |
| 67 | for _ in range(depth): |
| 68 | self.layers.append( |
| 69 | nn.ModuleList( |
| 70 | [MultiQueryAttention(dim, heads), FeedForward(dim, mlp_dim)] |
| 71 | ) |
| 72 | ) |
| 73 | |
| 74 | def forward(self, x): |
| 75 | for attn, ff in self.layers: |
| 76 | x, _, _ = attn(x) |
| 77 | x = self.norm(x) + x |
| 78 | x = ff(x) + x |
| 79 | return self.norm(x) |
| 80 | |
| 81 | |
| 82 | class OneBitViT(nn.Module): |