| 39 | |
| 40 | |
| 41 | class Glove: |
| 42 | def __init__(self, D, V, context_sz): |
| 43 | self.D = D |
| 44 | self.V = V |
| 45 | self.context_sz = context_sz |
| 46 | |
| 47 | def fit(self, sentences, cc_matrix=None, learning_rate=1e-4, reg=0.1, xmax=100, alpha=0.75, epochs=10, gd=False, use_theano=False, use_tensorflow=False): |
| 48 | # build co-occurrence matrix |
| 49 | # paper calls it X, so we will call it X, instead of calling |
| 50 | # the training data X |
| 51 | # TODO: would it be better to use a sparse matrix? |
| 52 | t0 = datetime.now() |
| 53 | V = self.V |
| 54 | D = self.D |
| 55 | |
| 56 | if not os.path.exists(cc_matrix): |
| 57 | X = np.zeros((V, V)) |
| 58 | N = len(sentences) |
| 59 | print("number of sentences to process:", N) |
| 60 | it = 0 |
| 61 | for sentence in sentences: |
| 62 | it += 1 |
| 63 | if it % 10000 == 0: |
| 64 | print("processed", it, "/", N) |
| 65 | n = len(sentence) |
| 66 | for i in range(n): |
| 67 | # i is not the word index!!! |
| 68 | # j is not the word index!!! |
| 69 | # i just points to which element of the sequence (sentence) we're looking at |
| 70 | wi = sentence[i] |
| 71 | |
| 72 | start = max(0, i - self.context_sz) |
| 73 | end = min(n, i + self.context_sz) |
| 74 | |
| 75 | # we can either choose only one side as context, or both |
| 76 | # here we are doing both |
| 77 | |
| 78 | # make sure "start" and "end" tokens are part of some context |
| 79 | # otherwise their f(X) will be 0 (denominator in bias update) |
| 80 | if i - self.context_sz < 0: |
| 81 | points = 1.0 / (i + 1) |
| 82 | X[wi,0] += points |
| 83 | X[0,wi] += points |
| 84 | if i + self.context_sz > n: |
| 85 | points = 1.0 / (n - i) |
| 86 | X[wi,1] += points |
| 87 | X[1,wi] += points |
| 88 | |
| 89 | # left side |
| 90 | for j in range(start, i): |
| 91 | wj = sentence[j] |
| 92 | points = 1.0 / (i - j) # this is +ve |
| 93 | X[wi,wj] += points |
| 94 | X[wj,wi] += points |
| 95 | |
| 96 | # right side |
| 97 | for j in range(i + 1, end): |
| 98 | wj = sentence[j] |