| 26 | |
| 27 | |
| 28 | class GloveVectorizer: |
| 29 | def __init__(self): |
| 30 | # load in pre-trained word vectors |
| 31 | print('Loading word vectors...') |
| 32 | word2vec = {} |
| 33 | embedding = [] |
| 34 | idx2word = [] |
| 35 | with open('../large_files/glove.6B/glove.6B.50d.txt') as f: |
| 36 | # is just a space-separated text file in the format: |
| 37 | # word vec[0] vec[1] vec[2] ... |
| 38 | for line in f: |
| 39 | values = line.split() |
| 40 | word = values[0] |
| 41 | vec = np.asarray(values[1:], dtype='float32') |
| 42 | word2vec[word] = vec |
| 43 | embedding.append(vec) |
| 44 | idx2word.append(word) |
| 45 | print('Found %s word vectors.' % len(word2vec)) |
| 46 | |
| 47 | # save for later |
| 48 | self.word2vec = word2vec |
| 49 | self.embedding = np.array(embedding) |
| 50 | self.word2idx = {v:k for k,v in enumerate(idx2word)} |
| 51 | self.V, self.D = self.embedding.shape |
| 52 | |
| 53 | def fit(self, data): |
| 54 | pass |
| 55 | |
| 56 | def transform(self, data): |
| 57 | X = np.zeros((len(data), self.D)) |
| 58 | n = 0 |
| 59 | emptycount = 0 |
| 60 | for sentence in data: |
| 61 | tokens = sentence.lower().split() |
| 62 | vecs = [] |
| 63 | for word in tokens: |
| 64 | if word in self.word2vec: |
| 65 | vec = self.word2vec[word] |
| 66 | vecs.append(vec) |
| 67 | if len(vecs) > 0: |
| 68 | vecs = np.array(vecs) |
| 69 | X[n] = vecs.mean(axis=0) |
| 70 | else: |
| 71 | emptycount += 1 |
| 72 | n += 1 |
| 73 | print("Numer of samples with no words found: %s / %s" % (emptycount, len(data))) |
| 74 | return X |
| 75 | |
| 76 | def fit_transform(self, data): |
| 77 | self.fit(data) |
| 78 | return self.transform(data) |
| 79 | |
| 80 | |
| 81 | |