Returns a dictionary of (i, j) => float. For indices i and j in the given list of texts, the corresponding float is the percentage of text i that is also in text j. Overlap is measured by matching n-grams (by default, 5 successive words). An optional weight function
(texts=[], n=5, continuous=False, weight=lambda ngram: 1)
| 368 | return Weight(self / value, self.assessments) |
| 369 | |
| 370 | def intertextuality(texts=[], n=5, continuous=False, weight=lambda ngram: 1): |
| 371 | """ Returns a dictionary of (i, j) => float. |
| 372 | For indices i and j in the given list of texts, |
| 373 | the corresponding float is the percentage of text i that is also in text j. |
| 374 | Overlap is measured by matching n-grams (by default, 5 successive words). |
| 375 | An optional weight function can be used to supply the weight of each n-gram. |
| 376 | """ |
| 377 | map = {} # n-gram => text id's |
| 378 | sum = {} # text id => sum of weight(n-gram) |
| 379 | for i, txt in enumerate(texts): |
| 380 | for j, ngram in enumerate(ngrams(txt, n, continuous=continuous)): |
| 381 | if ngram not in map: |
| 382 | map[ngram] = set() |
| 383 | map[ngram].add(i) |
| 384 | sum[i] = sum.get(i, 0) + weight(ngram) |
| 385 | w = defaultdict(Weight) # (id1, id2) => percentage of id1 that overlaps with id2 |
| 386 | for ngram in map: |
| 387 | for i in map[ngram]: |
| 388 | for j in map[ngram]: |
| 389 | if i != j: |
| 390 | if (i,j) not in w: |
| 391 | w[i,j] = Weight(0.0) |
| 392 | w[i,j] += weight(ngram) |
| 393 | w[i,j].assessments.add(ngram) |
| 394 | for i, j in w: |
| 395 | w[i,j] /= float(sum[i]) |
| 396 | w[i,j] = min(w[i,j], Weight(1.0)) |
| 397 | return w |
| 398 | |
| 399 | #--- WORD TYPE-TOKEN RATIO ------------------------------------------------------------------------- |
| 400 |