| 307 | |
| 308 | |
| 309 | class T5Decoder(nn.Module): |
| 310 | |
| 311 | def __init__(self, |
| 312 | vocab, |
| 313 | dim, |
| 314 | dim_attn, |
| 315 | dim_ffn, |
| 316 | num_heads, |
| 317 | num_layers, |
| 318 | num_buckets, |
| 319 | shared_pos=True, |
| 320 | dropout=0.1): |
| 321 | super(T5Decoder, self).__init__() |
| 322 | self.dim = dim |
| 323 | self.dim_attn = dim_attn |
| 324 | self.dim_ffn = dim_ffn |
| 325 | self.num_heads = num_heads |
| 326 | self.num_layers = num_layers |
| 327 | self.num_buckets = num_buckets |
| 328 | self.shared_pos = shared_pos |
| 329 | |
| 330 | # layers |
| 331 | self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ |
| 332 | else nn.Embedding(vocab, dim) |
| 333 | self.pos_embedding = T5RelativeEmbedding( |
| 334 | num_buckets, num_heads, bidirectional=False) if shared_pos else None |
| 335 | self.dropout = nn.Dropout(dropout) |
| 336 | self.blocks = nn.ModuleList([ |
| 337 | T5CrossAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, |
| 338 | shared_pos, dropout) for _ in range(num_layers) |
| 339 | ]) |
| 340 | self.norm = T5LayerNorm(dim) |
| 341 | |
| 342 | # initialize weights |
| 343 | self.apply(init_weights) |
| 344 | |
| 345 | def forward(self, ids, mask=None, encoder_states=None, encoder_mask=None): |
| 346 | b, s = ids.size() |
| 347 | |
| 348 | # causal mask |
| 349 | if mask is None: |
| 350 | mask = torch.tril(torch.ones(1, s, s).to(ids.device)) |
| 351 | elif mask.ndim == 2: |
| 352 | mask = torch.tril(mask.unsqueeze(1).expand(-1, s, -1)) |
| 353 | |
| 354 | # layers |
| 355 | x = self.token_embedding(ids) |
| 356 | x = self.dropout(x) |
| 357 | e = self.pos_embedding(x.size(1), |
| 358 | x.size(1)) if self.shared_pos else None |
| 359 | for block in self.blocks: |
| 360 | x = block(x, mask, encoder_states, encoder_mask, pos_bias=e) |
| 361 | x = self.norm(x) |
| 362 | x = self.dropout(x) |
| 363 | return x |
| 364 | |
| 365 | |
| 366 | class T5Model(nn.Module): |