(
self,
bpe_path: Optional[str] = None,
additional_special_tokens: Optional[List[str]] = None,
context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH,
clean: str = 'lower',
)
| 145 | """Simple tokenizer for CLIP models.""" |
| 146 | |
| 147 | def __init__( |
| 148 | self, |
| 149 | bpe_path: Optional[str] = None, |
| 150 | additional_special_tokens: Optional[List[str]] = None, |
| 151 | context_length: Optional[int] = DEFAULT_CONTEXT_LENGTH, |
| 152 | clean: str = 'lower', |
| 153 | ): |
| 154 | if bpe_path is None: |
| 155 | bpe_path = default_bpe() |
| 156 | |
| 157 | if not os.path.exists(bpe_path): |
| 158 | raise FileNotFoundError( |
| 159 | f"BPE vocabulary file not found at {bpe_path}. " |
| 160 | "Please ensure the file exists or install open_clip." |
| 161 | ) |
| 162 | |
| 163 | self.byte_encoder = bytes_to_unicode() |
| 164 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} |
| 165 | |
| 166 | # Load BPE merges |
| 167 | merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') |
| 168 | merges = merges[1:49152-256-2+1] |
| 169 | merges = [tuple(merge.split()) for merge in merges] |
| 170 | |
| 171 | # Build vocabulary |
| 172 | vocab = list(bytes_to_unicode().values()) |
| 173 | vocab = vocab + [v+'</w>' for v in vocab] |
| 174 | for merge in merges: |
| 175 | vocab.append(''.join(merge)) |
| 176 | |
| 177 | special_tokens = ['<start_of_text>', '<end_of_text>'] |
| 178 | if additional_special_tokens: |
| 179 | special_tokens += additional_special_tokens |
| 180 | vocab.extend(special_tokens) |
| 181 | |
| 182 | self.encoder = dict(zip(vocab, range(len(vocab)))) |
| 183 | self.decoder = {v: k for k, v in self.encoder.items()} |
| 184 | self.bpe_ranks = dict(zip(merges, range(len(merges)))) |
| 185 | self.cache = {t: t for t in special_tokens} |
| 186 | |
| 187 | special = "|".join(special_tokens) |
| 188 | # Use regex with Unicode support if available, otherwise fallback to standard re |
| 189 | if HAS_REGEX: |
| 190 | self.pat = re.compile( |
| 191 | special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", |
| 192 | re.IGNORECASE, |
| 193 | ) |
| 194 | else: |
| 195 | # Fallback pattern if regex doesn't support \p{L} etc. |
| 196 | self.pat = re.compile( |
| 197 | special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[a-zA-Z]+|[0-9]+|[^\s\w]+""", |
| 198 | re.IGNORECASE, |
| 199 | ) |
| 200 | |
| 201 | self.vocab_size = len(self.encoder) |
| 202 | self.all_special_ids = [self.encoder[t] for t in special_tokens] |
| 203 | self.sot_token_id = self.all_special_ids[0] |
| 204 | self.eot_token_id = self.all_special_ids[1] |
nothing calls this directly
no test coverage detected