Function words

pattern/vector/__init__.py:203–212

Returns a list of words (alphanumeric character sequences) from the given string. Common punctuation marks are stripped from words.

(string, filter=lambda w: w.lstrip("'").isalnum(), punctuation=PUNCTUATION, **kwargs)

PUNCTUATION = ".,;:!?()[]{}`''\"@#$^&*+-|=~_"

def words(string, filter=lambda w: w.lstrip("'").isalnum(), punctuation=PUNCTUATION, **kwargs):
    """ Returns a list of words (alphanumeric character sequences) from the given string.
        Common punctuation marks are stripped from words.
    """
    string = decode_utf8(string)
    string = re.sub(r"([a-z|A-Z])'(m|s|ve|re|ll|d)", u"\\1 `'\\2", string)
    words = (w.strip(punctuation).replace(u"`'", "'", 1) for w in string.split())
    words = (w for w in words if filter is None or filter(w) is not False)
    words = [w for w in words if w]
    return words

PORTER, LEMMA = "porter", "lemma"
def stem(word, stemmer=PORTER, **kwargs):