(text)
| 56 | |
| 57 | # Turns text into list of vocabulary indices. |
| 58 | def encode(text): |
| 59 | R = [0] * config.MAX_LEN # all padding |
| 60 | text = tokenize(text) |
| 61 | for i in range(len(text)): |
| 62 | w = text[i] |
| 63 | if w in token2id: |
| 64 | R[i] = token2id[w] |
| 65 | else: |
| 66 | R[i] = 1 # OOV: [UNK] |
| 67 | return np.array(R) |
| 68 | |
| 69 | # Opposite of encode: restores text string from token ids. |
| 70 | def decode(tokens): |