Returns the tokenized representation of given input string(s) Parameters ---------- texts : Union[str, List[str]] An input string or a list of input strings to tokenize context_length : int The context length to use; all CLIP models use 77 as the context length
(texts: Union[str, List[str]], context_length: int = 77)
| 162 | |
| 163 | |
| 164 | def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor: |
| 165 | """ |
| 166 | Returns the tokenized representation of given input string(s) |
| 167 | |
| 168 | Parameters |
| 169 | ---------- |
| 170 | texts : Union[str, List[str]] |
| 171 | An input string or a list of input strings to tokenize |
| 172 | |
| 173 | context_length : int |
| 174 | The context length to use; all CLIP models use 77 as the context length |
| 175 | |
| 176 | Returns |
| 177 | ------- |
| 178 | A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] |
| 179 | """ |
| 180 | if isinstance(texts, str): |
| 181 | texts = [texts] |
| 182 | |
| 183 | sot_token = _tokenizer.encoder["<|startoftext|>"] |
| 184 | eot_token = _tokenizer.encoder["<|endoftext|>"] |
| 185 | all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] |
| 186 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) |
| 187 | |
| 188 | for i, tokens in enumerate(all_tokens): |
| 189 | if len(tokens) > context_length: |
| 190 | raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") |
| 191 | result[i, :len(tokens)] = torch.tensor(tokens) |
| 192 | |
| 193 | return result |