load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb :param cache_file_h5py: :param cache_file_pickle: :return:
(cache_file_h5py,cache_file_pickle)
| 9 | random_number=300 |
| 10 | |
| 11 | def load_data(cache_file_h5py,cache_file_pickle): |
| 12 | """ |
| 13 | load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb |
| 14 | :param cache_file_h5py: |
| 15 | :param cache_file_pickle: |
| 16 | :return: |
| 17 | """ |
| 18 | if not os.path.exists(cache_file_h5py) or not os.path.exists(cache_file_pickle): |
| 19 | raise RuntimeError("############################ERROR##############################\n. " |
| 20 | "please download cache file, it include training data and vocabulary & labels. " |
| 21 | "link can be found in README.md\n download zip file, unzip it, then put cache files as FLAGS." |
| 22 | "cache_file_h5py and FLAGS.cache_file_pickle suggested location.") |
| 23 | print("INFO. cache file exists. going to load cache file") |
| 24 | f_data = h5py.File(cache_file_h5py, 'r') |
| 25 | print("f_data.keys:",list(f_data.keys())) |
| 26 | train_X=f_data['train_X'] # np.array( |
| 27 | print("train_X.shape:",train_X.shape) |
| 28 | train_Y=f_data['train_Y'] # np.array( |
| 29 | print("train_Y.shape:",train_Y.shape,";") |
| 30 | vaild_X=f_data['vaild_X'] # np.array( |
| 31 | valid_Y=f_data['valid_Y'] # np.array( |
| 32 | test_X=f_data['test_X'] # np.array( |
| 33 | test_Y=f_data['test_Y'] # np.array( |
| 34 | #f_data.close() |
| 35 | |
| 36 | word2index, label2index=None,None |
| 37 | with open(cache_file_pickle, 'rb') as data_f_pickle: |
| 38 | word2index, label2index=pickle.load(data_f_pickle) |
| 39 | print("INFO. cache file load successful...") |
| 40 | return word2index, label2index,train_X,train_Y,vaild_X,valid_Y,test_X,test_Y |
| 41 | |
| 42 | ####################################### |
| 43 | def compute_f1_score(predict_y,eval_y): |