| 441 | return self.token_embedding(input_ids) + self.position_embedding(position_ids) |
| 442 | |
| 443 | class CLIPTextTransformer: |
| 444 | def __init__(self): |
| 445 | self.embeddings = CLIPTextEmbeddings() |
| 446 | self.encoder = CLIPEncoder() |
| 447 | self.final_layer_norm = LayerNorm(768) |
| 448 | |
| 449 | def __call__(self, input_ids): |
| 450 | seq_len = input_ids.shape[1] |
| 451 | x = self.embeddings(input_ids, Tensor.arange(seq_len).reshape(1, -1)) |
| 452 | mask = Tensor.full((1, 1, seq_len, seq_len), float("-inf")).triu(1) |
| 453 | x = self.encoder(x, mask) |
| 454 | return self.final_layer_norm(x) |
| 455 | |
| 456 | # Clip tokenizer, taken from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py (MIT license) |
| 457 | @lru_cache() |