()
| 128 | |
| 129 | |
| 130 | def init_tokenizer(): |
| 131 | # tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') |
| 132 | tokenizer = BertTokenizer.from_pretrained('/home/notebook/data/group/LowLevelLLM/LLM/bert-base-uncased', local_files_only=True) |
| 133 | tokenizer.add_special_tokens({'bos_token': '[DEC]'}) |
| 134 | tokenizer.add_special_tokens({'additional_special_tokens': ['[ENC]']}) |
| 135 | tokenizer.enc_token_id = tokenizer.additional_special_tokens_ids[0] |
| 136 | return tokenizer |
| 137 | |
| 138 | |
| 139 | def create_vit(vit, |