load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb :param cache_file_h5py: :param cache_file_pickle: :return:
(cache_file_h5py,cache_file_pickle)
| 163 | f.write(question_id+","+labels_string+"\n") |
| 164 | |
| 165 | def load_data(cache_file_h5py,cache_file_pickle): |
| 166 | """ |
| 167 | load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb |
| 168 | :param cache_file_h5py: |
| 169 | :param cache_file_pickle: |
| 170 | :return: |
| 171 | """ |
| 172 | if not os.path.exists(cache_file_h5py) or not os.path.exists(cache_file_pickle): |
| 173 | raise RuntimeError("############################ERROR##############################\n. " |
| 174 | "please download cache file, it include training data and vocabulary & labels. " |
| 175 | "link can be found in README.md\n download zip file, unzip it, then put cache files as FLAGS." |
| 176 | "cache_file_h5py and FLAGS.cache_file_pickle suggested location.") |
| 177 | print("INFO. cache file exists. going to load cache file") |
| 178 | f_data = h5py.File(cache_file_h5py, 'r') |
| 179 | print("f_data.keys:",list(f_data.keys())) |
| 180 | train_X=f_data['train_X'] # np.array( |
| 181 | print("train_X.shape:",train_X.shape) |
| 182 | train_Y=f_data['train_Y'] # np.array( |
| 183 | print("train_Y.shape:",train_Y.shape,";") |
| 184 | vaild_X=f_data['vaild_X'] # np.array( |
| 185 | valid_Y=f_data['valid_Y'] # np.array( |
| 186 | test_X=f_data['test_X'] # np.array( |
| 187 | test_Y=f_data['test_Y'] # np.array( |
| 188 | #f_data.close() |
| 189 | |
| 190 | word2index, label2index=None,None |
| 191 | with open(cache_file_pickle, 'rb') as data_f_pickle: |
| 192 | word2index, label2index=pickle.load(data_f_pickle) |
| 193 | print("INFO. cache file load successful...") |
| 194 | return word2index, label2index,train_X,train_Y,vaild_X,valid_Y,test_X,test_Y |
| 195 | |
| 196 | if __name__ == "__main__": |
| 197 | #tf.app.run() |
nothing calls this directly
no outgoing calls
no test coverage detected