Function tokenize

model-experiments/masked-language-model/infer.py:53–55

(text)

# Mimick the TextVectorization tokenization.
def tokenize(text):
    # split at ' '
    return text.split()

# Turns text into list of vocabulary indices.
def encode(text):