Function character_ngrams

pattern/vector/__init__.py:270–287

Returns a dictionary of (character n-gram, count)-items, in lowercase. N-grams in the exclude list are not counted. N-grams whose count falls below (or equals) the given threshold are excluded. N-grams that are not in the given top most counted are excluded.

(string="", n=3, top=None, threshold=0, exclude=[], **kwargs)

    return count

def character_ngrams(string="", n=3, top=None, threshold=0, exclude=[], **kwargs):
    """ Returns a dictionary of (character n-gram, count)-items, in lowercase.
        N-grams in the exclude list are not counted.
        N-grams whose count falls below (or equals) the given threshold are excluded.
        N-grams that are not in the given top most counted are excluded.
    """
    # An optional dict-parameter can be used to specify a subclass of dict, 
    # e.g., count(words, dict=readonlydict) as used in Document.
    count = kwargs.get("dict", dict)()
    for w in re.findall(r"(?=(" + "."*n + "))", string.lower()):
        if w not in exclude:
            dict.__setitem__(count, w, (w in count) and count[w]+1 or 1)
    for k in count.keys():
        if count[k] <= threshold:
            dict.__delitem__(count, k)
    if top is not None:
        count = count.__class__(heapq.nsmallest(top, count.iteritems(), key=lambda (k,v): (-v,k)))
    return count
    
chngrams = character_ngrams