Function build_tokenizer

training/modules/tokenizer.py:4–8

(args)

from transformers import AutoTokenizer, GPT2TokenizerFast, DebertaV2Tokenizer

def build_tokenizer(args):
    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name)
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    return tokenizer

def build_gpt2_tokenizer(args):
    tokenizer = GPT2TokenizerFast.from_pretrained(args.tokenizer_name)