()
| 27 | |
| 28 | |
| 29 | def main(): |
| 30 | analogies_to_try = ( |
| 31 | ('king', 'man', 'woman'), |
| 32 | ('france', 'paris', 'london'), |
| 33 | ('france', 'paris', 'rome'), |
| 34 | ('paris', 'france', 'italy'), |
| 35 | ) |
| 36 | |
| 37 | ### choose a data source ### |
| 38 | # sentences, word2idx = get_sentences_with_word2idx_limit_vocab(n_vocab=1500) |
| 39 | sentences, word2idx = get_wikipedia_data(n_files=3, n_vocab=2000, by_paragraph=True) |
| 40 | # with open('tfidf_word2idx.json', 'w') as f: |
| 41 | # json.dump(word2idx, f) |
| 42 | |
| 43 | notfound = False |
| 44 | for word_list in analogies_to_try: |
| 45 | for w in word_list: |
| 46 | if w not in word2idx: |
| 47 | print("%s not found in vocab, remove it from \ |
| 48 | analogies to try or increase vocab size" % w) |
| 49 | notfound = True |
| 50 | if notfound: |
| 51 | exit() |
| 52 | |
| 53 | |
| 54 | # build term document matrix |
| 55 | V = len(word2idx) |
| 56 | N = len(sentences) |
| 57 | |
| 58 | # create raw counts first |
| 59 | A = np.zeros((V, N)) |
| 60 | print("V:", V, "N:", N) |
| 61 | j = 0 |
| 62 | for sentence in sentences: |
| 63 | for i in sentence: |
| 64 | A[i,j] += 1 |
| 65 | j += 1 |
| 66 | print("finished getting raw counts") |
| 67 | |
| 68 | transformer = TfidfTransformer() |
| 69 | A = transformer.fit_transform(A.T).T |
| 70 | |
| 71 | # tsne requires a dense array |
| 72 | A = A.toarray() |
| 73 | |
| 74 | # map back to word in plot |
| 75 | idx2word = {v:k for k, v in iteritems(word2idx)} |
| 76 | |
| 77 | # plot the data in 2-D |
| 78 | tsne = TSNE() |
| 79 | Z = tsne.fit_transform(A) |
| 80 | plt.scatter(Z[:,0], Z[:,1]) |
| 81 | for i in range(V): |
| 82 | try: |
| 83 | plt.annotate(s=idx2word[i].encode("utf8").decode("utf8"), xy=(Z[i,0], Z[i,1])) |
| 84 | except: |
| 85 | print("bad string:", idx2word[i]) |
| 86 | plt.draw() |
no test coverage detected