helper function to mask `idx` token from `tokens` according to section 3.3.1 of https://arxiv.org/pdf/1810.04805.pdf
(self, idx, tokens, types, vocab_words, rng)
| 1063 | return (tokens_a, token_types_a), (tokens_b, token_types_b) |
| 1064 | |
| 1065 | def mask_token(self, idx, tokens, types, vocab_words, rng): |
| 1066 | """ |
| 1067 | helper function to mask `idx` token from `tokens` according to |
| 1068 | section 3.3.1 of https://arxiv.org/pdf/1810.04805.pdf |
| 1069 | """ |
| 1070 | label = tokens[idx] |
| 1071 | if rng.random() < 0.8: |
| 1072 | new_label = self.tokenizer.get_command('MASK').Id |
| 1073 | else: |
| 1074 | if rng.random() < 0.5: |
| 1075 | new_label = label |
| 1076 | else: |
| 1077 | new_label = rng.choice(vocab_words) |
| 1078 | |
| 1079 | tokens[idx] = new_label |
| 1080 | |
| 1081 | return label |
| 1082 | |
| 1083 | def pad_seq(self, seq): |
| 1084 | """helper function to pad sequence pair""" |
no test coverage detected