load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb :param cache_file_h5py: :param cache_file_pickle: :return:
(cache_file_h5py,cache_file_pickle)
| 220 | print("using pre-trained word emebedding.ended...") |
| 221 | |
| 222 | def load_data(cache_file_h5py,cache_file_pickle): |
| 223 | """ |
| 224 | load data from h5py and pickle cache files, which is generate by take step by step of pre-processing.ipynb |
| 225 | :param cache_file_h5py: |
| 226 | :param cache_file_pickle: |
| 227 | :return: |
| 228 | """ |
| 229 | if not os.path.exists(cache_file_h5py) or not os.path.exists(cache_file_pickle): |
| 230 | raise RuntimeError("############################ERROR##############################\n. " |
| 231 | "please download cache file, it include training data and vocabulary & labels. " |
| 232 | "link can be found in README.md\n download zip file, unzip it, then put cache files as FLAGS." |
| 233 | "cache_file_h5py and FLAGS.cache_file_pickle suggested location.") |
| 234 | print("INFO. cache file exists. going to load cache file") |
| 235 | f_data = h5py.File(cache_file_h5py, 'r') |
| 236 | print("f_data.keys:",list(f_data.keys())) |
| 237 | train_X=f_data['train_X'] # np.array( |
| 238 | print("train_X.shape:",train_X.shape) |
| 239 | train_Y=f_data['train_Y'] # np.array( |
| 240 | print("train_Y.shape:",train_Y.shape,";") |
| 241 | vaild_X=f_data['vaild_X'] # np.array( |
| 242 | valid_Y=f_data['valid_Y'] # np.array( |
| 243 | test_X=f_data['test_X'] # np.array( |
| 244 | test_Y=f_data['test_Y'] # np.array( |
| 245 | #print(train_X) |
| 246 | #f_data.close() |
| 247 | |
| 248 | word2index, label2index=None,None |
| 249 | with open(cache_file_pickle, 'rb') as data_f_pickle: |
| 250 | word2index, label2index=pickle.load(data_f_pickle) |
| 251 | print("INFO. cache file load successful...") |
| 252 | return word2index, label2index,train_X,train_Y,vaild_X,valid_Y,test_X,test_Y |
| 253 | if __name__ == "__main__": |
| 254 | tf.app.run() |