(config_encoder)
| 17 | |
| 18 | |
| 19 | def build_tokenizer(config_encoder): |
| 20 | tokenizer = None |
| 21 | os.environ['TOKENIZERS_PARALLELISM'] = 'true' |
| 22 | if config_encoder['TOKENIZER'] == 'clip': |
| 23 | pretrained_tokenizer = config_encoder.get( |
| 24 | 'PRETRAINED_TOKENIZER', 'openai/clip-vit-base-patch32' |
| 25 | ) |
| 26 | tokenizer = CLIPTokenizer.from_pretrained(pretrained_tokenizer) |
| 27 | tokenizer.add_special_tokens({'cls_token': tokenizer.eos_token}) |
| 28 | elif config_encoder['TOKENIZER'] == 'clip-fast': |
| 29 | pretrained_tokenizer = config_encoder.get( |
| 30 | 'PRETRAINED_TOKENIZER', 'openai/clip-vit-base-patch32' |
| 31 | ) |
| 32 | tokenizer = CLIPTokenizerFast.from_pretrained(pretrained_tokenizer, from_slow=True) |
| 33 | else: |
| 34 | tokenizer = AutoTokenizer.from_pretrained(config_encoder['TOKENIZER']) |
| 35 | |
| 36 | return tokenizer |
no test coverage detected