Function tokenize

model-experiments/masked-language-model/test.py:72–74

(text)

# Mimick the TextVectorization tokenization.
def tokenize(text):
    # split at ' '
    return text.split()

# Turns text into list of vocabulary indices.
def encode(text):