Function get_code_data_eval

codegeex/mindspore/src/dataset.py:201–232

(code_data_path, args_opt, process_fn=None, scale=1)

def get_code_data_eval(code_data_path, args_opt, process_fn=None, scale=1):
    datasets = []
    paths = []

    for dir in sorted(os.listdir(code_data_path)):
        sub_dirs = os.listdir(os.path.join(code_data_path, dir))
        for sub_dir in sub_dirs:
            if os.path.exists(os.path.join(code_data_path, dir, sub_dir, 'data.mdb')) and os.path.exists(
                    os.path.join(code_data_path, dir, sub_dir, 'lock.mdb')):
                paths.append(os.path.join(code_data_path, dir, sub_dir))

    for full_path in paths:
        if os.path.isdir(full_path):
            print(f"Loading code data {full_path}")
            data = LMDBDataset(
                full_path,
                process_fn=process_fn,
            )
            data = PadDataset(
                data,
                args_opt.seq_length,
                args_opt.eod_id,
            )
            data = SubsetDataset(
                data,
                0,
                len(data) // 10,
            )
            datasets.append(data)
    datasets = ConcatDataset(datasets)
    print(f"==valid dataset has {len(datasets)} items")
    return datasets