(use_cache=True)
| 174 | |
| 175 | |
| 176 | def prepare_data(use_cache=True): |
| 177 | import pickle |
| 178 | if not os.path.isfile(insuranceQA_cache_fp) or not use_cache: |
| 179 | # no cache is found, preprocess data from scratch |
| 180 | print("prepare data from scratch") |
| 181 | |
| 182 | # get pretained word vector |
| 183 | from gensim.models.keyedvectors import KeyedVectors |
| 184 | google_news_pretrain_fp = check_exist_or_download( |
| 185 | google_news_pretrain_embeddings_link) |
| 186 | wv = KeyedVectors.load_word2vec_format(google_news_pretrain_fp, |
| 187 | binary=True) |
| 188 | |
| 189 | # prepare insurance QA dataset |
| 190 | data_zip = check_exist_or_download(insuranceQA_url) |
| 191 | data_dir = unzip_data(download_dir, data_zip) |
| 192 | |
| 193 | label2answer = get_label2answer(data_dir) |
| 194 | idx2word = get_idx2word(data_dir) |
| 195 | idx2vec = get_idx2vec_weights(wv, idx2word) |
| 196 | |
| 197 | train_raw = get_train_raw(data_dir, insuranceqa_train_filename) |
| 198 | test_raw = get_train_raw(data_dir, insuranceqa_test_filename) |
| 199 | with open(insuranceQA_cache_fp, 'wb') as handle: |
| 200 | pickle.dump((train_raw, test_raw, label2answer, idx2word, idx2vec), |
| 201 | handle, |
| 202 | protocol=pickle.HIGHEST_PROTOCOL) |
| 203 | else: |
| 204 | # load from cached pickle |
| 205 | with open(insuranceQA_cache_fp, 'rb') as handle: |
| 206 | (train_raw, test_raw, label2answer, idx2word, |
| 207 | idx2vec) = pickle.load(handle) |
| 208 | |
| 209 | return train_raw, test_raw, label2answer, idx2word, idx2vec |
| 210 | |
| 211 | |
| 212 | def limit_encode_eval(train_raw, |
no test coverage detected