| 67 | |
| 68 | |
| 69 | def get_wiki(): |
| 70 | V = 20000 |
| 71 | files = glob('../large_files/enwiki*.txt') |
| 72 | all_word_counts = {} |
| 73 | for f in files: |
| 74 | for line in open(f): |
| 75 | if line and line[0] not in '[*-|=\{\}': |
| 76 | s = remove_punctuation(line).lower().split() |
| 77 | if len(s) > 1: |
| 78 | for word in s: |
| 79 | if word not in all_word_counts: |
| 80 | all_word_counts[word] = 0 |
| 81 | all_word_counts[word] += 1 |
| 82 | print("finished counting") |
| 83 | |
| 84 | V = min(V, len(all_word_counts)) |
| 85 | all_word_counts = sorted(all_word_counts.items(), key=lambda x: x[1], reverse=True) |
| 86 | |
| 87 | top_words = [w for w, count in all_word_counts[:V-1]] + ['<UNK>'] |
| 88 | word2idx = {w:i for i, w in enumerate(top_words)} |
| 89 | unk = word2idx['<UNK>'] |
| 90 | |
| 91 | sents = [] |
| 92 | for f in files: |
| 93 | for line in open(f): |
| 94 | if line and line[0] not in '[*-|=\{\}': |
| 95 | s = remove_punctuation(line).lower().split() |
| 96 | if len(s) > 1: |
| 97 | # if a word is not nearby another word, there won't be any context! |
| 98 | # and hence nothing to train! |
| 99 | sent = [word2idx[w] if w in word2idx else unk for w in s] |
| 100 | sents.append(sent) |
| 101 | return sents, word2idx |
| 102 | |
| 103 | |
| 104 | |