Function load_tokenizer

examples/utils.py:199–212

(tokenizer_dir: Optional[str] = None,
                   vocab_file: Optional[str] = None,
                   model_name: str = 'GPTForCausalLM',
                   model_version: Optional[str] = None,
                   tokenizer_type: Optional[str] = None)

def load_tokenizer(tokenizer_dir: Optional[str] = None,
                   vocab_file: Optional[str] = None,
                   model_name: str = 'GPTForCausalLM',
                   model_version: Optional[str] = None,
                   tokenizer_type: Optional[str] = None):
    func = partial(_load_tokenizer, tokenizer_dir, vocab_file, model_name,
                   model_version, tokenizer_type)
    if mpi_world_size() > 1:
        # Under MPI env, load tokenizer will result in multiple processes to download the same file to the same folder.
        # This will result some random bug. Force loading on rank0 to warmup the tokenizer to avoid this issue.
        if mpi_rank() == 0:
            func()
        mpi_barrier()
    return func()


def prepare_enc_dec_inputs(batch_input_ids: List[torch.Tensor], model_name: str,