Returns the tokenized representation of given input string(s) Parameters ---------- texts : Union[str, List[str]] An input string or a list of input strings to tokenize context_length : int The context length to use; all CLIP models use 77 as the context length
(texts: Union[str, List[str]],
context_length: int = 77,
truncate: bool = False)
| 44 | |
| 45 | |
| 46 | def tokenize(texts: Union[str, List[str]], |
| 47 | context_length: int = 77, |
| 48 | truncate: bool = False) -> torch.LongTensor: |
| 49 | """ |
| 50 | Returns the tokenized representation of given input string(s) |
| 51 | |
| 52 | Parameters |
| 53 | ---------- |
| 54 | texts : Union[str, List[str]] |
| 55 | An input string or a list of input strings to tokenize |
| 56 | |
| 57 | context_length : int |
| 58 | The context length to use; all CLIP models use 77 as the context length |
| 59 | |
| 60 | truncate: bool |
| 61 | Whether to truncate the text in case its encoding is longer than the context length |
| 62 | |
| 63 | Returns |
| 64 | ------- |
| 65 | A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] |
| 66 | """ |
| 67 | if isinstance(texts, str): |
| 68 | texts = [texts] |
| 69 | |
| 70 | sot_token = _tokenizer.encoder["<|startoftext|>"] |
| 71 | eot_token = _tokenizer.encoder["<|endoftext|>"] |
| 72 | all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] |
| 73 | for text in texts] |
| 74 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) |
| 75 | |
| 76 | for i, tokens in enumerate(all_tokens): |
| 77 | if len(tokens) > context_length: |
| 78 | if truncate: |
| 79 | tokens = tokens[:context_length] |
| 80 | tokens[-1] = eot_token |
| 81 | else: |
| 82 | raise RuntimeError( |
| 83 | f"Input {texts[i]} is too long for context length {context_length}" |
| 84 | ) |
| 85 | result[i, :len(tokens)] = torch.tensor(tokens) |
| 86 | |
| 87 | return result |
| 88 | |
| 89 | |
| 90 | def loads_pyarrow(buf): |