Function encode

model-experiments/masked-language-model/train.py:128–130

(texts)

# Encodes the token strings by int vocab indices.
def encode(texts):
    encoded_texts = vectorize_layer(texts)
    return encoded_texts.numpy()

# Randomly replace tokens by the [mask] and keep replaced token as label.
def get_masked_input_and_labels(encoded_texts):