| 75 | |
| 76 | # Turns text into list of vocabulary indices. |
| 77 | def encode(text): |
| 78 | R = [0] * config.MAX_LEN # all padding |
| 79 | text = tokenize(text) |
| 80 | ntoks = len(text) |
| 81 | if ntoks > config.MAX_LEN: |
| 82 | ntoks = config.MAX_LEN |
| 83 | text = text[:ntoks] |
| 84 | # pick random position (never a padding): |
| 85 | k = random.randint(0, ntoks-1) |
| 86 | golden = 0 |
| 87 | for i in range(len(text)): |
| 88 | w = text[i] |
| 89 | if w in token2id: |
| 90 | R[i] = token2id[w] |
| 91 | else: |
| 92 | R[i] = 1 # OOV: [UNK] |
| 93 | if i == k: |
| 94 | #golden = w |
| 95 | golden = R[i] |
| 96 | #print("k:", k, "golden:", golden) |
| 97 | R[i] = mask_token_id |
| 98 | return k, golden, np.array(R) |
| 99 | |
| 100 | def predict(text): |
| 101 | k, golden, R = encode(text) |