Function stem

pattern/vector/stemmer.py:314–343

Returns the stem of the given word: ponies => poni. Note: it is often taken to be a crude error that a stemming algorithm does not leave a real word after removing the stem. But the purpose of stemming is to bring variant forms of a word together, not to map a wor

(word, cached=True, history=10000, **kwargs)

cache = {}

def stem(word, cached=True, history=10000, **kwargs):
    """ Returns the stem of the given word: ponies => poni.
        Note: it is often taken to be a crude error 
        that a stemming algorithm does not leave a real word after removing the stem. 
        But the purpose of stemming is to bring variant forms of a word together, 
        not to map a word onto its "paradigm" form. 
    """
    stem = word.lower()
    if cached and stem in cache:
        return case_sensitive(cache[stem], word)
    if cached and len(cache) > history: # Empty cache every now and then.
        cache.clear()
    if len(stem) <= 2:
        # If the word has two letters or less, leave it as it is. 
        return case_sensitive(stem, word)
    if stem in exceptions:
        return case_sensitive(exceptions[stem], word)
    if stem in uninflected:
        return case_sensitive(stem, word)
    # Mark y treated as a consonant as Y.
    stem = upper_consonant_y(stem)
    for f in (step_1a, step_1b, step_1c, step_2, step_3, step_4, step_5a, step_5b):
        stem = f(stem)
    # Turn any remaining Y letters in the stem back into lower case.
    # Apply the case of the original word to the stem. 
    stem = stem.lower()
    stem = case_sensitive(stem, word)
    if cached:
        cache[word.lower()] = stem.lower()
    return stem