(self, embed_dim=768, vocab_size=49408, max_position_embeddings=77, num_encoder_layers=12, encoder_intermediate_size=3072)
| 38 | |
| 39 | class SDTextEncoder(torch.nn.Module): |
| 40 | def __init__(self, embed_dim=768, vocab_size=49408, max_position_embeddings=77, num_encoder_layers=12, encoder_intermediate_size=3072): |
| 41 | super().__init__() |
| 42 | |
| 43 | # token_embedding |
| 44 | self.token_embedding = torch.nn.Embedding(vocab_size, embed_dim) |
| 45 | |
| 46 | # position_embeds (This is a fixed tensor) |
| 47 | self.position_embeds = torch.nn.Parameter(torch.zeros(1, max_position_embeddings, embed_dim)) |
| 48 | |
| 49 | # encoders |
| 50 | self.encoders = torch.nn.ModuleList([CLIPEncoderLayer(embed_dim, encoder_intermediate_size) for _ in range(num_encoder_layers)]) |
| 51 | |
| 52 | # attn_mask |
| 53 | self.attn_mask = self.attention_mask(max_position_embeddings) |
| 54 | |
| 55 | # final_layer_norm |
| 56 | self.final_layer_norm = torch.nn.LayerNorm(embed_dim) |
| 57 | |
| 58 | def attention_mask(self, length): |
| 59 | mask = torch.empty(length, length) |
no test coverage detected