Function tokenize_words

numpy_ml/preprocessing/nlp.py:77–86

Split a string into individual words, optionally removing punctuation and stop-words in the process.

(
    line, lowercase=True, filter_stopwords=True, filter_punctuation=True, **kwargs,
)

def tokenize_words(
    line, lowercase=True, filter_stopwords=True, filter_punctuation=True, **kwargs,
):
    """
    Split a string into individual words, optionally removing punctuation and
    stop-words in the process.
    """
    REGEX = _WORD_REGEX if filter_punctuation else _WORD_REGEX_W_PUNC
    words = REGEX.findall(line.lower() if lowercase else line)
    return remove_stop_words(words) if filter_stopwords else words


def tokenize_words_bytes(