| 72 | |
| 73 | # Do predictions: |
| 74 | def predict(text): |
| 75 | sample = np.reshape(encode(text), (1, config.MAX_LEN)) |
| 76 | print("sample.shape:", sample.shape) #(1, 256) |
| 77 | prediction = mlm_model.predict(sample) |
| 78 | print("prediction.shape:", prediction.shape) #(1, 256, 512) |
| 79 | |
| 80 | # position of token2id['[mask]'] in list: |
| 81 | masked_index = np.where(sample == mask_token_id)[1][0] |
| 82 | print("masked_index:", masked_index) |
| 83 | # all substitute word probabilities: |
| 84 | mask_prediction = prediction[0][masked_index] |
| 85 | #mask_prediction.shape = (512,) |
| 86 | top_k = 1 |
| 87 | # word indices with top-k highest probabilities: |
| 88 | # Trick: negate array so order is reversed: |
| 89 | #top_indices = (-mask_prediction).argsort()[0:top_k] |
| 90 | top_indices = mask_prediction.argsort()[-top_k:][::-1] |
| 91 | # probabilities of the top_k |
| 92 | values = mask_prediction[top_indices] |
| 93 | |
| 94 | for i in range(len(top_indices)): |
| 95 | w = id2token[top_indices[i]] |
| 96 | v = values[i] |
| 97 | # fill in the blank: |
| 98 | #tokens = np.copy(sample[0]) |
| 99 | #tokens[masked_index] = p |
| 100 | result = { |
| 101 | "input_text": text, |
| 102 | # better use original text: |
| 103 | "prediction": text.replace('[mask]', w), |
| 104 | "probability": v, |
| 105 | #"predicted mask token": w, |
| 106 | } |
| 107 | pprint(result) |
| 108 | |
| 109 | # Read a sentence from stdin, 1 per line: |
| 110 | import sys |