Runs basic whitespace cleaning and splitting on a piece of text.
(text)
| 106 | |
| 107 | |
| 108 | def whitespace_tokenize(text): |
| 109 | """Runs basic whitespace cleaning and splitting on a piece of text.""" |
| 110 | text = text.strip() |
| 111 | if not text: |
| 112 | return [] |
| 113 | tokens = text.split() |
| 114 | return tokens |
| 115 | |
| 116 | |
| 117 | class BertTokenizer(PreTrainedTokenizer): |