Returns the tokenized representation of given input string(s) Parameters ---------- texts : Union[str, List[str]] An input string or a list of input strings to tokenize context_length : int The context length to use; all CLIP models use 77 as the context length
(texts: Union[str, List[str]],
context_length: int = 77,
truncate: bool = False)
| 41 | |
| 42 | |
| 43 | def tokenize(texts: Union[str, List[str]], |
| 44 | context_length: int = 77, |
| 45 | truncate: bool = False) -> torch.LongTensor: |
| 46 | """ |
| 47 | Returns the tokenized representation of given input string(s) |
| 48 | |
| 49 | Parameters |
| 50 | ---------- |
| 51 | texts : Union[str, List[str]] |
| 52 | An input string or a list of input strings to tokenize |
| 53 | |
| 54 | context_length : int |
| 55 | The context length to use; all CLIP models use 77 as the context length |
| 56 | |
| 57 | truncate: bool |
| 58 | Whether to truncate the text in case its encoding is longer than the context length |
| 59 | |
| 60 | Returns |
| 61 | ------- |
| 62 | A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] |
| 63 | """ |
| 64 | l_mask = [0] * context_length |
| 65 | result = [0] * context_length |
| 66 | |
| 67 | tokens = _tokenizer.encode(text=texts, add_special_tokens=True) |
| 68 | tokens = tokens[:context_length] |
| 69 | result[:len(tokens)] = tokens |
| 70 | l_mask[:len(tokens)] = [1]*len(tokens) |
| 71 | |
| 72 | result = torch.tensor(result).unsqueeze(0) |
| 73 | l_mask = torch.tensor(l_mask).unsqueeze(0) |
| 74 | return result, l_mask |
| 75 | |
| 76 | |
| 77 | def loads_pyarrow(buf): |