| 45 | |
| 46 | |
| 47 | def get_wiki(): |
| 48 | V = 20000 |
| 49 | files = glob('../large_files/enwiki*.txt') |
| 50 | all_word_counts = {} |
| 51 | for f in files: |
| 52 | for line in open(f): |
| 53 | if line and line[0] not in '[*-|=\{\}': |
| 54 | s = remove_punctuation(line).lower().split() |
| 55 | if len(s) > 1: |
| 56 | for word in s: |
| 57 | if word not in all_word_counts: |
| 58 | all_word_counts[word] = 0 |
| 59 | all_word_counts[word] += 1 |
| 60 | print("finished counting") |
| 61 | |
| 62 | V = min(V, len(all_word_counts)) |
| 63 | all_word_counts = sorted(all_word_counts.items(), key=lambda x: x[1], reverse=True) |
| 64 | |
| 65 | top_words = [w for w, count in all_word_counts[:V-1]] + ['<UNK>'] |
| 66 | word2idx = {w:i for i, w in enumerate(top_words)} |
| 67 | unk = word2idx['<UNK>'] |
| 68 | |
| 69 | sents = [] |
| 70 | for f in files: |
| 71 | for line in open(f): |
| 72 | if line and line[0] not in '[*-|=\{\}': |
| 73 | s = remove_punctuation(line).lower().split() |
| 74 | if len(s) > 1: |
| 75 | # if a word is not nearby another word, there won't be any context! |
| 76 | # and hence nothing to train! |
| 77 | sent = [word2idx[w] if w in word2idx else unk for w in s] |
| 78 | sents.append(sent) |
| 79 | return sents, word2idx |
| 80 | |
| 81 | |
| 82 | |