collect and preprocess raw data from acl Imdb dataset
()
| 176 | |
| 177 | |
| 178 | def preprocess(): |
| 179 | ''' collect and preprocess raw data from acl Imdb dataset |
| 180 | ''' |
| 181 | nltk.download('stopwords') |
| 182 | |
| 183 | print("preparing raw imdb data") |
| 184 | data_gz = check_exist_or_download(imdb_dataset_link) |
| 185 | data_dir = unzip_data(download_dir, data_gz) |
| 186 | |
| 187 | # imdb dirs |
| 188 | # vocab_f = data_dir + '/imdb.vocab' |
| 189 | train_pos_dir = data_dir + '/train/pos/' |
| 190 | train_neg_dir = data_dir + '/train/neg/' |
| 191 | test_pos_dir = data_dir + '/test/pos/' |
| 192 | test_neg_dir = data_dir + '/test/neg/' |
| 193 | |
| 194 | # nltk helpers |
| 195 | tokenizer = ToktokTokenizer() |
| 196 | stopword_list = nltk.corpus.stopwords.words('english') |
| 197 | |
| 198 | # load pretrained word2vec binary |
| 199 | print("loading pretrained word2vec") |
| 200 | google_news_pretrain_fp = check_exist_or_download( |
| 201 | google_news_pretrain_embeddings_link) |
| 202 | wv = KeyedVectors.load_word2vec_format(google_news_pretrain_fp, binary=True) |
| 203 | |
| 204 | # parse flat files to memory |
| 205 | data = [] |
| 206 | for data_dir, label in [(train_pos_dir, 1), (train_neg_dir, 0), |
| 207 | (test_pos_dir, 1), (test_neg_dir, 0)]: |
| 208 | for filename in os.listdir(data_dir): |
| 209 | if filename.endswith(".txt"): |
| 210 | with open(os.path.join(data_dir, filename), |
| 211 | "r", |
| 212 | encoding="utf-8") as fhdl: |
| 213 | data.append((fhdl.read(), label)) |
| 214 | |
| 215 | # text review cleaning |
| 216 | print("cleaning text review") |
| 217 | imdb_data = pd.DataFrame(data, columns=["review", "label"]) |
| 218 | imdb_data['review'] = imdb_data['review'].apply(strip_html) |
| 219 | imdb_data['review'] = imdb_data['review'].apply( |
| 220 | remove_between_square_brackets) |
| 221 | imdb_data['review'] = imdb_data['review'].apply(remove_special_characters) |
| 222 | imdb_data['review'] = imdb_data['review'].apply(simple_stemmer) |
| 223 | imdb_data['review'] = imdb_data['review'].apply(remove_stopwords, |
| 224 | args=(tokenizer, |
| 225 | stopword_list)) |
| 226 | imdb_data['token'] = imdb_data['review'].apply(tokenize) |
| 227 | |
| 228 | # build word2index and index2word |
| 229 | w2i = dict() |
| 230 | i2w = dict() |
| 231 | |
| 232 | # add vocab <pad> as index 0 |
| 233 | w2i["<pad>"] = 0 |
| 234 | i2w[0] = "<pad>" |
| 235 |
no test coverage detected