Function load_dataset

src/process_data.py:11–44

r"""A dataset loader using a global config. It will load the train, valid, and test set (if exists) simulatenously.

(dataset)

from openprompt.data_utils.data_processor import DataProcessor

def load_dataset(dataset):
    r"""A dataset loader using a global config.
    It will load the train, valid, and test set (if exists) simulatenously.
    
    """

    processor = PROCESSORS[dataset.lower()]()
    path = 'datasets/' + dataset.lower().split('-')[0]

    train_dataset = None
    valid_dataset = None
    
    try:
        train_dataset = processor.get_train_examples(path)
    except FileNotFoundError:
        logger.warning(f"Has no training dataset in {path}.")
    try:
        valid_dataset = processor.get_dev_examples(path)
    except FileNotFoundError:
        logger.warning(f"Has no validation dataset in {path}.")

    test_dataset = None
    try:
        test_dataset = processor.get_test_examples(path)
    except FileNotFoundError:
        logger.warning(f"Has no test dataset in {path}.")
    # checking whether donwloaded.
    if (train_dataset is None) and \
       (valid_dataset is None) and \
       (test_dataset is None):
        logger.error("Dataset is empty. Either there is no download or the path is wrong. "+ \
        "If not downloaded, please `cd datasets/` and `bash download_xxx.sh`")
        exit()
    return train_dataset, valid_dataset, test_dataset, processor
        

class MnlimmProcessor(DataProcessor):