| 25 | |
| 26 | |
| 27 | class Glove: |
| 28 | def __init__(self, D, V, context_sz): |
| 29 | self.D = D |
| 30 | self.V = V |
| 31 | self.context_sz = context_sz |
| 32 | |
| 33 | def fit(self, sentences, cc_matrix=None): |
| 34 | # build co-occurrence matrix |
| 35 | # paper calls it X, so we will call it X, instead of calling |
| 36 | # the training data X |
| 37 | # TODO: would it be better to use a sparse matrix? |
| 38 | t0 = datetime.now() |
| 39 | V = self.V |
| 40 | D = self.D |
| 41 | |
| 42 | if not os.path.exists(cc_matrix): |
| 43 | X = np.zeros((V, V)) |
| 44 | N = len(sentences) |
| 45 | print("number of sentences to process:", N) |
| 46 | it = 0 |
| 47 | for sentence in sentences: |
| 48 | it += 1 |
| 49 | if it % 10000 == 0: |
| 50 | print("processed", it, "/", N) |
| 51 | n = len(sentence) |
| 52 | for i in range(n): |
| 53 | # i is not the word index!!! |
| 54 | # j is not the word index!!! |
| 55 | # i just points to which element of the sequence (sentence) we're looking at |
| 56 | wi = sentence[i] |
| 57 | |
| 58 | start = max(0, i - self.context_sz) |
| 59 | end = min(n, i + self.context_sz) |
| 60 | |
| 61 | # we can either choose only one side as context, or both |
| 62 | # here we are doing both |
| 63 | |
| 64 | # make sure "start" and "end" tokens are part of some context |
| 65 | # otherwise their f(X) will be 0 (denominator in bias update) |
| 66 | if i - self.context_sz < 0: |
| 67 | points = 1.0 / (i + 1) |
| 68 | X[wi,0] += points |
| 69 | X[0,wi] += points |
| 70 | if i + self.context_sz > n: |
| 71 | points = 1.0 / (n - i) |
| 72 | X[wi,1] += points |
| 73 | X[1,wi] += points |
| 74 | |
| 75 | # left side |
| 76 | for j in range(start, i): |
| 77 | wj = sentence[j] |
| 78 | points = 1.0 / (i - j) # this is +ve |
| 79 | X[wi,wj] += points |
| 80 | X[wj,wi] += points |
| 81 | |
| 82 | # right side |
| 83 | for j in range(i + 1, end): |
| 84 | wj = sentence[j] |