| 270 | |
| 271 | |
| 272 | class T5Encoder(nn.Module): |
| 273 | |
| 274 | def __init__(self, |
| 275 | vocab, |
| 276 | dim, |
| 277 | dim_attn, |
| 278 | dim_ffn, |
| 279 | num_heads, |
| 280 | num_layers, |
| 281 | num_buckets, |
| 282 | shared_pos=True, |
| 283 | dropout=0.1): |
| 284 | super(T5Encoder, self).__init__() |
| 285 | self.dim = dim |
| 286 | self.dim_attn = dim_attn |
| 287 | self.dim_ffn = dim_ffn |
| 288 | self.num_heads = num_heads |
| 289 | self.num_layers = num_layers |
| 290 | self.num_buckets = num_buckets |
| 291 | self.shared_pos = shared_pos |
| 292 | |
| 293 | # layers |
| 294 | self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ |
| 295 | else nn.Embedding(vocab, dim) |
| 296 | self.pos_embedding = T5RelativeEmbedding( |
| 297 | num_buckets, num_heads, bidirectional=True) if shared_pos else None |
| 298 | self.dropout = nn.Dropout(dropout) |
| 299 | self.blocks = nn.ModuleList([ |
| 300 | T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, |
| 301 | shared_pos, dropout) for _ in range(num_layers) |
| 302 | ]) |
| 303 | self.norm = T5LayerNorm(dim) |
| 304 | |
| 305 | # initialize weights |
| 306 | self.apply(init_weights) |
| 307 | |
| 308 | def forward(self, ids, mask=None): |
| 309 | x = self.token_embedding(ids) |
| 310 | x = self.dropout(x) |
| 311 | e = self.pos_embedding(x.size(1), |
| 312 | x.size(1)) if self.shared_pos else None |
| 313 | for block in self.blocks: |
| 314 | x = block(x, mask, pos_bias=e) |
| 315 | x = self.norm(x) |
| 316 | x = self.dropout(x) |
| 317 | return x |
| 318 | |
| 319 | |
| 320 | class T5Decoder(nn.Module): |