(self, data)
| 262 | return reflect_ids |
| 263 | |
| 264 | def __call__(self, data): |
| 265 | text = data['label'] |
| 266 | text = self.encode(text) |
| 267 | if text is None: |
| 268 | return None |
| 269 | data['length'] = np.array(len(text) + 1) |
| 270 | text = text + [self.dict[self.EOS]] |
| 271 | p_mask_list = [] |
| 272 | noisy_batch_list = [] |
| 273 | masked_indices_list = [] |
| 274 | reflect_ids_list = [] |
| 275 | for i in range(self.sample_num): |
| 276 | reflect_ids = self.reflect_random_idices(text) |
| 277 | reflect_ids = reflect_ids + [self.dict[self.MASK]] * ( |
| 278 | self.max_text_len + 1 - len(reflect_ids)) |
| 279 | if self.semi_ar: |
| 280 | noisy_batch, masked_indices = self.forward_process_semi_ar( |
| 281 | text) |
| 282 | else: |
| 283 | noisy_batch, masked_indices = self.forward_process(text) |
| 284 | p_mask = float(sum(masked_indices)) / float(len(text)) |
| 285 | p_mask_list.append(np.array(p_mask)) |
| 286 | noisy_batch_list.append(np.array(noisy_batch)) |
| 287 | masked_indices_list.append(np.array(masked_indices)) |
| 288 | reflect_ids_list.append(np.array(reflect_ids)) |
| 289 | |
| 290 | if not self.semi_ar: |
| 291 | data['p_mask'] = np.array( |
| 292 | p_mask_list) if self.train_all_layer else np.array( |
| 293 | p_mask_list[0]) |
| 294 | data['noisy_batch'] = np.array( |
| 295 | noisy_batch_list) if self.train_all_layer else np.array( |
| 296 | noisy_batch_list[0]) |
| 297 | data['masked_indices'] = np.array( |
| 298 | masked_indices_list) if self.train_all_layer else np.array( |
| 299 | masked_indices_list[0]) |
| 300 | data['reflect_ids'] = np.array( |
| 301 | reflect_ids_list) if self.train_all_layer else np.array( |
| 302 | reflect_ids_list[0]) |
| 303 | |
| 304 | text = text + [self.dict[self.PAD] |
| 305 | ] * (self.max_text_len + 1 - len(text)) |
| 306 | data['label'] = np.array(text) |
| 307 | |
| 308 | return data |
| 309 | |
| 310 | def add_special_char(self, dict_character): |
| 311 | dict_character = [self.EOS] + dict_character + [self.MASK, self.PAD] |
nothing calls this directly
no test coverage detected