Load all eval dataset
(eval_args: EvaluateArguments)
| 10 | |
| 11 | |
| 12 | def load_all_dataset(eval_args: EvaluateArguments): |
| 13 | ''' |
| 14 | Load all eval dataset |
| 15 | ''' |
| 16 | # get fp for eval dataset |
| 17 | dataset_name_list = eval_args.eval_dataset_list |
| 18 | eval_dataset_fp_conf_path = eval_args.eval_dataset_fp_conf_path |
| 19 | |
| 20 | with open(eval_dataset_fp_conf_path, 'r') as f: |
| 21 | dataset_fn_dict = json.load(f) |
| 22 | |
| 23 | data_dir = eval_args.data_path |
| 24 | |
| 25 | logger.info(dataset_name_list) |
| 26 | if len(dataset_name_list) == 1 and dataset_name_list[0] == 'all': |
| 27 | dataset_name_list = dataset_fn_dict.keys() |
| 28 | dataset_fp_list = [data_dir + os.path.sep + eval_args.eval_language + os.path.sep + eval_args.eval_dataset_type + os.path.sep + dataset_fn_dict[i] for i in dataset_name_list] |
| 29 | |
| 30 | logger.info('Start load and preprocess dataset') |
| 31 | all_dataset = {} |
| 32 | for dataset_name in dataset_name_list: |
| 33 | dataset_fp = data_dir + os.path.sep + eval_args.eval_language + os.path.sep + eval_args.eval_dataset_type + os.path.sep + dataset_fn_dict[dataset_name] |
| 34 | df = pd.read_csv(dataset_fp) |
| 35 | |
| 36 | # Read dev data if doing few-shot test |
| 37 | df_dev = None |
| 38 | if eval_args.k_shot > 0: |
| 39 | dev_dataset_fp = data_dir + os.path.sep + eval_args.eval_language + os.path.sep + 'dev' + os.path.sep + dataset_fn_dict[dataset_name] |
| 40 | df_dev = pd.read_csv(dev_dataset_fp) |
| 41 | |
| 42 | all_dataset[dataset_name] = preprocess(df, eval_args, df_dev=df_dev) |
| 43 | logger.info('Load success, dataset_name={}, dataset_file_path={}, dataset question count={}'.format(dataset_name, |
| 44 | dataset_fp, |
| 45 | len(all_dataset[dataset_name]))) |
| 46 | return all_dataset |
| 47 | |
| 48 | if __name__ == '__main__': |
| 49 | a = os.path.split(os.path.realpath(__file__))[0] |
no test coverage detected