(self, idx)
| 820 | return len(self.example_list) |
| 821 | |
| 822 | def __getitem__(self, idx): |
| 823 | example = self.example_list[idx] |
| 824 | cls_id = self.tokenizer.get_command('ENC').Id |
| 825 | mask_token = 'sMASK' if self.task_mask else 'MASK' |
| 826 | mask_id = self.tokenizer.get_command(mask_token).Id |
| 827 | eos_id = self.tokenizer.get_command('eos').Id |
| 828 | pad_id = self.tokenizer.get_command('pad').Id |
| 829 | sop_id = self.tokenizer.get_command('sop').Id |
| 830 | eop_id = self.tokenizer.get_command('eop').Id |
| 831 | source_text, target_text = example.text_a, example.text_b |
| 832 | source_tokens = self.tokenizer.EncodeAsIds(source_text).tokenization |
| 833 | if len(source_tokens) + 3 > self.max_src_length: |
| 834 | source_tokens = source_tokens[-(self.max_src_length - 3):] |
| 835 | source_tokens = [cls_id] + source_tokens + [mask_id, eos_id] |
| 836 | context_length = len(source_tokens) |
| 837 | if len(source_tokens) < self.max_src_length: |
| 838 | source_tokens = source_tokens + [pad_id] * (self.max_src_length - len(source_tokens)) |
| 839 | sep = len(source_tokens) |
| 840 | position_ids = list(range(len(source_tokens))) |
| 841 | block_position_ids = [0] * len(source_tokens) |
| 842 | mask_pos = source_tokens.index(mask_id) |
| 843 | if self.split == 'train': |
| 844 | target_tokens = self.tokenizer.EncodeAsIds(" " + target_text).tokenization |
| 845 | target_tokens = target_tokens + [eop_id] |
| 846 | if len(target_tokens) > self.max_tgt_length: |
| 847 | target_tokens = target_tokens[:self.max_tgt_length] |
| 848 | loss_mask = [1] * len(target_tokens) |
| 849 | if len(target_tokens) < self.max_tgt_length: |
| 850 | loss_mask += [0] * (self.max_tgt_length - len(target_tokens)) |
| 851 | target_tokens += [pad_id] * (self.max_tgt_length - len(target_tokens)) |
| 852 | if self.mask_pad_token: |
| 853 | attention_mask = np.zeros((self.max_src_length + self.max_tgt_length, |
| 854 | self.max_src_length + self.max_tgt_length), dtype=np.int64) |
| 855 | attention_mask[:, :context_length] = 1 |
| 856 | attention_mask[self.max_src_length:, self.max_src_length:] = np.tril( |
| 857 | np.ones((self.max_tgt_length, self.max_tgt_length), dtype=np.int64)) |
| 858 | attention_mask = attention_mask[None, :, :] |
| 859 | else: |
| 860 | attention_mask = np.array(sep, dtype=np.int64) |
| 861 | tokens = source_tokens + [sop_id] + target_tokens[:-1] |
| 862 | loss_mask = [0] * len(source_tokens) + loss_mask |
| 863 | target_ids = [0] * len(source_tokens) + target_tokens |
| 864 | position_ids += [mask_pos] * len(target_tokens) |
| 865 | if self.no_block_position: |
| 866 | block_position_ids += [1] * len(target_tokens) |
| 867 | else: |
| 868 | block_position_ids += list(range(1, len(target_tokens) + 1)) |
| 869 | position_ids = [position_ids, block_position_ids] |
| 870 | sample = {'text': np.array(tokens, dtype=np.int64), 'target': np.array(target_ids, dtype=np.int64), |
| 871 | 'attention_mask': attention_mask, |
| 872 | 'loss_mask': np.array(loss_mask, dtype=np.int64), |
| 873 | "position_id": np.array(position_ids, dtype=np.int64), "uid": example.guid} |
| 874 | else: |
| 875 | tokens = source_tokens + [sop_id] |
| 876 | position_ids = position_ids + [mask_pos] |
| 877 | block_position_ids = block_position_ids + [1] |
| 878 | position_ids = [position_ids, block_position_ids] |
| 879 | if self.mask_pad_token: |
nothing calls this directly
no test coverage detected