Function get_tokenizer

data_utils/unified_tokenizer.py:198–206

(args=None)

        return ret

def get_tokenizer(args=None):
    if not hasattr(get_tokenizer, 'tokenizer'):
        # the first time to load the tokenizer, specify img_tokenizer_path
        get_tokenizer.tokenizer = UnifiedTokenizer(
            args.img_tokenizer_path, 
            device=torch.cuda.current_device(),
            img_tokenizer_num_tokens=args.img_tokenizer_num_tokens
            )
    return get_tokenizer.tokenizer

class FakeTokenizer(object):
    def __init__(self, num_tokens):