Get tokenizer instance. Args: model_name: Model name (for compatibility, not used in simple implementation) context_length: Context length for tokenizer cache_dir: Cache directory (for compatibility, not used) **kwargs: Additional arguments passed to SimpleT
(
model_name: str = 'ViT-B-32',
context_length: Optional[int] = None,
cache_dir: Optional[str] = None,
**kwargs
)
| 295 | |
| 296 | |
| 297 | def get_tokenizer( |
| 298 | model_name: str = 'ViT-B-32', |
| 299 | context_length: Optional[int] = None, |
| 300 | cache_dir: Optional[str] = None, |
| 301 | **kwargs |
| 302 | ): |
| 303 | """ |
| 304 | Get tokenizer instance. |
| 305 | |
| 306 | Args: |
| 307 | model_name: Model name (for compatibility, not used in simple implementation) |
| 308 | context_length: Context length for tokenizer |
| 309 | cache_dir: Cache directory (for compatibility, not used) |
| 310 | **kwargs: Additional arguments passed to SimpleTokenizer |
| 311 | |
| 312 | Returns: |
| 313 | SimpleTokenizer instance |
| 314 | """ |
| 315 | context_length = context_length or DEFAULT_CONTEXT_LENGTH |
| 316 | |
| 317 | # Try to use open_clip's tokenizer if available |
| 318 | try: |
| 319 | from open_clip import get_tokenizer as _get_tokenizer |
| 320 | return _get_tokenizer(model_name, context_length=context_length, cache_dir=cache_dir, **kwargs) |
| 321 | except ImportError: |
| 322 | pass |
| 323 | |
| 324 | # Use our simple tokenizer |
| 325 | return SimpleTokenizer(context_length=context_length, **kwargs) |