Function build_dataset_from_file

modelscope/trainers/training_args.py:501–563

The filename format: [ { "dataset": { "dataset_name": "xxx", ... }, "column_mapping": { "text1": "sequence1", "text2": "sequence2", "label": "label", }

(filename)

def build_dataset_from_file(filename):
    """
    The filename format:
    [
        {
            "dataset": {
                "dataset_name": "xxx",
                ...
            },
            "column_mapping": {
                "text1": "sequence1",
                "text2": "sequence2",
                "label": "label",
            }
            "usage": 0.8,
        }
    ]
    """
    from modelscope import MsDataset
    train_set = []
    eval_set = []

    with open(filename, 'r') as f:
        ds_json = json.load(f)
        for ds in ds_json:
            dataset = MsDataset.load(**ds['dataset']).to_hf_dataset()
            all_columns = dataset.column_names
            keep_columns = ds['column_mapping'].keys()
            remove_columns = [
                column for column in all_columns if column not in keep_columns
            ]
            from datasets import Features
            from datasets import Value
            from datasets import ClassLabel
            features = [
                f for f in dataset.features.items() if f[0] in keep_columns
            ]
            new_features = {}
            for f in features:
                if isinstance(f[1], ClassLabel):
                    new_features[f[0]] = Value(f[1].dtype)
                else:
                    new_features[f[0]] = f[1]
            new_features = Features(new_features)
            dataset = dataset.map(
                lambda x: x,
                remove_columns=remove_columns,
                features=new_features).rename_columns(ds['column_mapping'])
            usage = ds['usage']
            if isinstance(usage, str):
                assert usage in ('train', 'val')
                if usage == 'train':
                    train_set.append(dataset)
                else:
                    eval_set.append(dataset)
            else:
                assert isinstance(usage, float) and 0 < usage < 1
                ds_dict = dataset.train_test_split(train_size=usage)