load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb :param cache_file_h5py: :param cache_file_pickle: :return:
(cache_file_h5py,cache_file_pickle)
| 128 | |
| 129 | |
| 130 | def load_data(cache_file_h5py,cache_file_pickle): |
| 131 | """ |
| 132 | load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb |
| 133 | :param cache_file_h5py: |
| 134 | :param cache_file_pickle: |
| 135 | :return: |
| 136 | """ |
| 137 | if not os.path.exists(cache_file_h5py) or not os.path.exists(cache_file_pickle): |
| 138 | raise RuntimeError("############################ERROR##############################\n. " |
| 139 | "please download cache file, it include training data and vocabulary & labels. " |
| 140 | "link can be found in README.md\n download zip file, unzip it, then put cache files as FLAGS." |
| 141 | "cache_file_h5py and FLAGS.cache_file_pickle suggested location.") |
| 142 | print("INFO. cache file exists. going to load cache file") |
| 143 | f_data = h5py.File(cache_file_h5py, 'r') |
| 144 | print("f_data.keys:",list(f_data.keys())) |
| 145 | train_X=f_data['train_X'] # np.array( |
| 146 | print("train_X.shape:",train_X.shape) |
| 147 | train_Y=f_data['train_Y'] # np.array( |
| 148 | print("train_Y.shape:",train_Y.shape,";") |
| 149 | vaild_X=f_data['vaild_X'] # np.array( |
| 150 | valid_Y=f_data['valid_Y'] # np.array( |
| 151 | test_X=f_data['test_X'] # np.array( |
| 152 | test_Y=f_data['test_Y'] # np.array( |
| 153 | #print(train_X) |
| 154 | #f_data.close() |
| 155 | |
| 156 | word2index, label2index=None,None |
| 157 | with open(cache_file_pickle, 'rb') as data_f_pickle: |
| 158 | word2index, label2index=pickle.load(data_f_pickle) |
| 159 | print("INFO. cache file load successful...") |
| 160 | return word2index, label2index,train_X,train_Y,vaild_X,valid_Y,test_X,test_Y |
| 161 | |
| 162 | #training_data_path='../data/sample_multiple_label3.txt' |
| 163 | #vocab_size=100 |