If you're using outer_tokenizer, call `get_tokenizer(args, outer_tokenizer)` before `training_main`.
(args=None, *, tokenizer_type=None, outer_tokenizer=None)
| 17 | |
| 18 | |
| 19 | def get_tokenizer(args=None, *, tokenizer_type=None, outer_tokenizer=None): |
| 20 | ''' |
| 21 | If you're using outer_tokenizer, call `get_tokenizer(args, outer_tokenizer)` |
| 22 | before `training_main`. |
| 23 | ''' |
| 24 | if outer_tokenizer is not None: # set 1 |
| 25 | get_tokenizer.tokenizer = outer_tokenizer |
| 26 | get_tokenizer.tokenizer_type = 'outer_tokenizer' |
| 27 | print_rank0('> Set tokenizer as an outer_tokenizer! Now you can get_tokenizer() everywhere.') |
| 28 | return outer_tokenizer |
| 29 | if tokenizer_type is None: |
| 30 | if args is None: |
| 31 | assert hasattr(get_tokenizer, 'tokenizer'), 'Never set tokenizer.' |
| 32 | return get_tokenizer.tokenizer |
| 33 | tokenizer_type = args.tokenizer_type |
| 34 | |
| 35 | # find the tokenizer via tokenizer_type! |
| 36 | if hasattr(get_tokenizer, 'tokenizer_type') and \ |
| 37 | tokenizer_type == get_tokenizer.tokenizer_type: # the same as last |
| 38 | return get_tokenizer.tokenizer |
| 39 | |
| 40 | get_tokenizer.tokenizer_type = tokenizer_type |
| 41 | # load the tokenizer according to tokenizer_type |
| 42 | if tokenizer_type.startswith('cogview'): # or cogview_ICE |
| 43 | from .cogview import UnifiedTokenizer |
| 44 | get_tokenizer.tokenizer = UnifiedTokenizer( |
| 45 | args.img_tokenizer_path, |
| 46 | txt_tokenizer_type='cogview', |
| 47 | device=torch.cuda.current_device() |
| 48 | ) |
| 49 | elif tokenizer_type.startswith('glm'): |
| 50 | kwargs = {"add_block_symbols": True, "add_task_mask": args.task_mask, |
| 51 | "add_decoder_mask": args.block_mask_prob > 0.0} |
| 52 | if tokenizer_type == "glm_GPT2BPETokenizer": |
| 53 | from .glm import GPT2BPETokenizer |
| 54 | get_tokenizer.tokenizer = GPT2BPETokenizer(args.tokenizer_model_type, **kwargs) |
| 55 | elif tokenizer_type == "glm_ChineseSPTokenizer": |
| 56 | from .glm import ChineseSPTokenizer |
| 57 | get_tokenizer.tokenizer = ChineseSPTokenizer(args.tokenizer_model_type, **kwargs) |
| 58 | elif tokenizer_type == "glm_BertWordPieceTokenizer": |
| 59 | from .glm import BertWordPieceTokenizer |
| 60 | get_tokenizer.tokenizer = BertWordPieceTokenizer(args.tokenizer_model_type, **kwargs) |
| 61 | elif tokenizer_type == 'icetk': |
| 62 | from icetk import icetk |
| 63 | get_tokenizer.tokenizer = icetk |
| 64 | elif tokenizer_type == 'icetk-glm-130B': |
| 65 | from .icetk_glm_130B import _IceTokenizer |
| 66 | get_tokenizer.tokenizer = _IceTokenizer() |
| 67 | # elif tokenizer_type.startswith('hf'): |
| 68 | # from .hf_tokenizer import HFT5Tokenizer |
| 69 | # if tokenizer_type == "hf_T5Tokenizer": |
| 70 | # get_tokenizer.tokenizer = HFT5Tokenizer(args.tokenizer_model_type, cache_dir=args.cache_dir) |
| 71 | else: |
| 72 | print_rank0('Try to load tokenizer from Huggingface transformers...') |
| 73 | os.environ['TOKENIZERS_PARALLELISM'] = 'true' |
| 74 | from transformers import AutoTokenizer |
| 75 | try: |
| 76 | get_tokenizer.tokenizer = AutoTokenizer.from_pretrained(tokenizer_type, trust_remote_code=True) |
no test coverage detected