Returns the tokenized representation of given input string(s). Parameters ---------- texts : Union[str, List[str]] An input string or a list of input strings to tokenize context_length : int The context length to use; all CLIP models use 77 as
(self, texts: Union[str, List[str]], context_length: Optional[int] = None)
| 263 | return text |
| 264 | |
| 265 | def __call__(self, texts: Union[str, List[str]], context_length: Optional[int] = None) -> torch.LongTensor: |
| 266 | """Returns the tokenized representation of given input string(s). |
| 267 | |
| 268 | Parameters |
| 269 | ---------- |
| 270 | texts : Union[str, List[str]] |
| 271 | An input string or a list of input strings to tokenize |
| 272 | context_length : int |
| 273 | The context length to use; all CLIP models use 77 as the context length |
| 274 | |
| 275 | Returns |
| 276 | ------- |
| 277 | A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] |
| 278 | """ |
| 279 | if isinstance(texts, str): |
| 280 | texts = [texts] |
| 281 | |
| 282 | context_length = context_length or self.context_length |
| 283 | assert context_length, 'Please set a valid context length' |
| 284 | |
| 285 | all_tokens = [[self.sot_token_id] + self.encode(text) + [self.eot_token_id] for text in texts] |
| 286 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) |
| 287 | |
| 288 | for i, tokens in enumerate(all_tokens): |
| 289 | if len(tokens) > context_length: |
| 290 | tokens = tokens[:context_length] # Truncate |
| 291 | tokens[-1] = self.eot_token_id |
| 292 | result[i, :len(tokens)] = torch.tensor(tokens) |
| 293 | |
| 294 | return result |
| 295 | |
| 296 | |
| 297 | def get_tokenizer( |