| 120 | |
| 121 | |
| 122 | class CLIPEmbeddings(torch.nn.Module): |
| 123 | def __init__(self, embed_dim, vocab_size=49408, num_positions=77, dtype=None, device=None): |
| 124 | super().__init__() |
| 125 | self.token_embedding = torch.nn.Embedding(vocab_size, embed_dim, dtype=dtype, device=device) |
| 126 | self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device) |
| 127 | |
| 128 | def forward(self, input_tokens): |
| 129 | return self.token_embedding(input_tokens) + self.position_embedding.weight |
| 130 | |
| 131 | |
| 132 | class CLIPTextModel_(torch.nn.Module): |