Function whitespace_tokenize

data_utils/wordpiece.py:65–71

Runs basic whitespace cleaning and splitting on a piece of text.

(text)

def whitespace_tokenize(text):
    """Runs basic whitespace cleaning and splitting on a piece of text."""
    text = text.strip()
    if not text:
        return []
    tokens = text.split()
    return tokens


class BertTokenizer(object):