Method load

fast_bert/data.py:335–348

(data_dir, backend='nccl', filename="databunch.pkl")

    @staticmethod
    def load(data_dir, backend='nccl', filename="databunch.pkl"):

        try:
            torch.distributed.init_process_group(backend=backend,
                                                 init_method="tcp://localhost:23459",
                                                 rank=0, world_size=1)
        except:
            pass

        tmp_path = data_dir/'tmp'
        with open(str(tmp_path/filename), "rb") as f:
            databunch = pickle.load(f)

        return databunch

    def __init__(self, data_dir, label_dir, tokenizer, train_file='train.csv', val_file='val.csv', test_data=None,
                 label_file='labels.csv', text_col='text', label_col='label', bs=32, maxlen=512,