Function cooccurrence

pattern/metrics.py:461–510

Returns the co-occurence matrix of terms in the given iterable as a dictionary: {term1: {term2: count, term3: count, ...}}. A string can be given as an iterable over the words with: isplit(string). A file can be given as an iterable over the words with: isplit(chain(*open("f

(iterable, window=(-1,-1), match=lambda x: False, filter=lambda x: True, normalize=lambda x: x)

    if a: yield "".join(a)

def cooccurrence(iterable, window=(-1,-1), match=lambda x: False, filter=lambda x: True, normalize=lambda x: x):
    """ Returns the co-occurence matrix of terms in the given iterable
        as a dictionary: {term1: {term2: count, term3: count, ...}}.
        A string can be given as an iterable over the words with: isplit(string).
        A file can be given as an iterable over the words with: isplit(chain(*open("file.txt"))).
        The given match() function determines search terms.
        The given filter() function determines co-occurring terms to count.
        The given normalize() function can be used to remove punctuation, lowercase words, etc.
        The given window, a (before, after)-tuple, specifies the size of the co-occurence window.
    """
    class Sentinel(object):
        pass
    # Window of terms before and after the search term.
    # Deque is more efficient than list.pop(0).
    q = deque()
    # Window size of terms alongside the search term.
    # Note that window=(0,0) will return a dictionary of search term frequency
    # (since it counts co-occurence with itself).
    n = -min(0, window[0]) + max(window[1], 0)
    m = {}
    # Search terms may fall outside the co-occurrence window, e.g., window=(-3,-2).
    # We add sentinel markers at the start and end of the given iterable.
    for x in chain([Sentinel()] * n, iterable, [Sentinel()] * n):
        q.append(x)
        if len(q) > n:
            # Given window q size and offset,
            # find the index of the candidate term:
            if window[1] >= 0:
                i = -1 - window[1]
            if window[1] < 0:
                i = len(q) - 1
            if i < 0:
                i = len(q) + i
            x1 = q[i]
            if not isinstance(x1, Sentinel):
                x1 = normalize(x1)
                if match(x1):
                    # Iterate the window and filter co-occurent terms.
                    for x2 in list(q).__getslice__(i+window[0], i+window[1]+1):
                        if not isinstance(x2, Sentinel):
                            x2 = normalize(x2)
                            if filter(x2):
                                if x1 not in m:
                                    m[x1] = {}
                                if x2 not in m[x1]:
                                    m[x1][x2] = 0
                                m[x1][x2] += 1
            # Slide window.
            q.popleft()
    return m
    
co_occurrence = cooccurrence