(self, text)
| 195 | return noisy_batch, masked_indices |
| 196 | |
| 197 | def forward_process(self, text): |
| 198 | |
| 199 | rand_choice = random.choice(self.mask_tpye) |
| 200 | if rand_choice == 0: # 并行mask full mask |
| 201 | return self.full_mask(text) |
| 202 | elif rand_choice == 1 and len(text) > 2: # 正向自回归 right mask |
| 203 | return self.left_to_right_mask(text) |
| 204 | elif rand_choice == 2 and len(text) > 2: # 反向自回归 left mask |
| 205 | return self.right_to_left_mask(text) |
| 206 | elif rand_choice == 3 and len(text) > 2: # block mask |
| 207 | rand_step = min(random.randint(2, 6), len(text)) |
| 208 | if rand_step <= 1: # len(text) <= 1 |
| 209 | return self.full_mask(text) |
| 210 | block_size = len(text) // rand_step |
| 211 | if block_size == 1: |
| 212 | return self.left_to_right_mask(text) if random.random( |
| 213 | ) < 0.5 else self.right_to_left_mask(text) |
| 214 | # 余数处理 |
| 215 | if len(text) % rand_step != 0: |
| 216 | rand_step += 1 |
| 217 | # 选择一个随机的block_size |
| 218 | rand_step_from_mask = random.randint(2, rand_step) |
| 219 | if rand_step == 2: |
| 220 | rand_step_from_mask = 1 |
| 221 | else: |
| 222 | rand_step_from_mask = random.randint(2, rand_step) |
| 223 | |
| 224 | noisy_batch = text[:block_size * (rand_step_from_mask - 1)] |
| 225 | masked_indices = [False] * (block_size * (rand_step_from_mask - 1)) |
| 226 | |
| 227 | noisy_batch = noisy_batch + [self.dict[self.MASK]] * ( |
| 228 | self.max_text_len + 1 - len(noisy_batch)) |
| 229 | masked_indices = masked_indices + [True] * ( |
| 230 | len(text) - block_size * |
| 231 | (rand_step_from_mask - 1)) + [False] * (self.max_text_len + 1 - |
| 232 | len(text)) |
| 233 | return noisy_batch, masked_indices |
| 234 | elif rand_choice == 4 and len(text) > 2: # cloze mask |
| 235 | noisy_batch = text[:] |
| 236 | masked_indices = [False] * len(text) |
| 237 | rand_index = random.randint(0, len(text) - 1) |
| 238 | noisy_batch[rand_index] = self.dict[self.MASK] |
| 239 | masked_indices[rand_index] = True |
| 240 | noisy_batch = noisy_batch + [self.dict[self.MASK]] * ( |
| 241 | self.max_text_len + 1 - len(text)) |
| 242 | masked_indices = masked_indices + [False] * (self.max_text_len + |
| 243 | 1 - len(text)) |
| 244 | return noisy_batch, masked_indices |
| 245 | else: # random mask |
| 246 | # 随机将text中的部分token mask掉 |
| 247 | noisy_batch, masked_indices = self.random_mask(text) |
| 248 | noisy_batch = noisy_batch + [self.dict[self.MASK]] * ( |
| 249 | self.max_text_len + 1 - len(text)) |
| 250 | masked_indices = masked_indices + [False] * (self.max_text_len + |
| 251 | 1 - len(text)) |
| 252 | return noisy_batch, masked_indices |
| 253 | |
| 254 | def reflect_random_idices(self, text, eps=1e-3): |
no test coverage detected