Build Text vectorization layer Args: texts (list): List of string, i.e., input texts vocab_size (int): vocab size max_seq (int): Maximum sequence length. Returns: layers.Layer: Return TextVectorization Keras Layer
(texts, vocab_size, max_seq)
| 74 | return input_data |
| 75 | |
| 76 | def get_vectorize_layer(texts, vocab_size, max_seq): |
| 77 | """Build Text vectorization layer |
| 78 | |
| 79 | Args: |
| 80 | texts (list): List of string, i.e., input texts |
| 81 | vocab_size (int): vocab size |
| 82 | max_seq (int): Maximum sequence length. |
| 83 | |
| 84 | Returns: |
| 85 | layers.Layer: Return TextVectorization Keras Layer |
| 86 | """ |
| 87 | vectorize_layer = TextVectorization( |
| 88 | max_tokens=vocab_size, |
| 89 | output_mode="int", |
| 90 | standardize=custom_standardization, |
| 91 | output_sequence_length=max_seq, |
| 92 | ) |
| 93 | vectorize_layer.adapt(texts) |
| 94 | |
| 95 | # Insert mask token in vocabulary |
| 96 | vocab = vectorize_layer.get_vocabulary() |
| 97 | #print("len(vocab):", len(vocab)) #177 |
| 98 | #vocab: ['', '[UNK]', 'the', 'and', 'a', 'of', ...] all lower-case |
| 99 | #GJ20: where do the empty string and [UNK] come from? |
| 100 | # they are created by adapt() as words 0 and 1 |
| 101 | # '' is padding token; [UNK] is OOV token |
| 102 | vocab = vocab[2:len(vocab)-1] + ["[mask]"] |
| 103 | #print("len(vocab):", len(vocab)) #175 |
| 104 | #GJ20: anyway first 2 words removed and '[mask]' added at the end |
| 105 | vectorize_layer.set_vocabulary(vocab) |
| 106 | # '' and [UNK] are back in |
| 107 | #vocab = vectorize_layer.get_vocabulary() |
| 108 | #print("len(vocab):", len(vocab)) #177 |
| 109 | # '[mask]' has been added as last (least frequent) word in the vocab |
| 110 | return vectorize_layer |
| 111 | |
| 112 | vectorize_layer = get_vectorize_layer( |
| 113 | all_data.tokens.values.tolist(), |