| 28 | |
| 29 | |
| 30 | class Glove: |
| 31 | def __init__(self, D, V, context_sz): |
| 32 | self.D = D |
| 33 | self.V = V |
| 34 | self.context_sz = context_sz |
| 35 | |
| 36 | def fit(self, sentences, cc_matrix=None, learning_rate=1e-4, reg=0.1, xmax=100, alpha=0.75, epochs=10): |
| 37 | # build co-occurrence matrix |
| 38 | # paper calls it X, so we will call it X, instead of calling |
| 39 | # the training data X |
| 40 | # TODO: would it be better to use a sparse matrix? |
| 41 | t0 = datetime.now() |
| 42 | V = self.V |
| 43 | D = self.D |
| 44 | |
| 45 | if not os.path.exists(cc_matrix): |
| 46 | X = np.zeros((V, V)) |
| 47 | N = len(sentences) |
| 48 | print("number of sentences to process:", N) |
| 49 | it = 0 |
| 50 | for sentence in sentences: |
| 51 | it += 1 |
| 52 | if it % 10000 == 0: |
| 53 | print("processed", it, "/", N) |
| 54 | n = len(sentence) |
| 55 | for i in range(n): |
| 56 | # i is not the word index!!! |
| 57 | # j is not the word index!!! |
| 58 | # i just points to which element of the sequence (sentence) we're looking at |
| 59 | wi = sentence[i] |
| 60 | |
| 61 | start = max(0, i - self.context_sz) |
| 62 | end = min(n, i + self.context_sz) |
| 63 | |
| 64 | # we can either choose only one side as context, or both |
| 65 | # here we are doing both |
| 66 | |
| 67 | # make sure "start" and "end" tokens are part of some context |
| 68 | # otherwise their f(X) will be 0 (denominator in bias update) |
| 69 | if i - self.context_sz < 0: |
| 70 | points = 1.0 / (i + 1) |
| 71 | X[wi,0] += points |
| 72 | X[0,wi] += points |
| 73 | if i + self.context_sz > n: |
| 74 | points = 1.0 / (n - i) |
| 75 | X[wi,1] += points |
| 76 | X[1,wi] += points |
| 77 | |
| 78 | # left side |
| 79 | for j in range(start, i): |
| 80 | wj = sentence[j] |
| 81 | points = 1.0 / (i - j) # this is +ve |
| 82 | X[wi,wj] += points |
| 83 | X[wj,wi] += points |
| 84 | |
| 85 | # right side |
| 86 | for j in range(i + 1, end): |
| 87 | wj = sentence[j] |