Function whitespace_tokenize

tokenizations/tokenization_bert_word_level.py:80–86

Runs basic whitespace cleaning and splitting on a piece of text.

(text)

def whitespace_tokenize(text):
    """Runs basic whitespace cleaning and splitting on a piece of text."""
    text = text.strip()
    if not text:
        return []
    tokens = text.split()
    return tokens


class BertTokenizer(PreTrainedTokenizer):