(self, example: InputExample, labelled: bool, priming: bool = False,
**kwargs)
| 45 | """Preprocessor for models pretrained using a masked language modeling objective (e.g., BERT).""" |
| 46 | |
| 47 | def get_input_features(self, example: InputExample, labelled: bool, priming: bool = False, |
| 48 | **kwargs) -> InputFeatures: |
| 49 | |
| 50 | input_ids, token_type_ids, block_flag = self.pvp.encode(example) |
| 51 | |
| 52 | attention_mask = [1] * len(input_ids) |
| 53 | padding_length = self.wrapper.config.max_seq_length - len(input_ids) |
| 54 | |
| 55 | if padding_length < 0: |
| 56 | raise ValueError(f"Maximum sequence length is too small, got {len(input_ids)} input ids") |
| 57 | |
| 58 | input_ids = input_ids + ([self.wrapper.tokenizer.pad_token_id] * padding_length) |
| 59 | attention_mask = attention_mask + ([0] * padding_length) |
| 60 | token_type_ids = token_type_ids + ([0] * padding_length) |
| 61 | block_flag = block_flag + ([0] * padding_length) |
| 62 | |
| 63 | assert len(input_ids) == self.wrapper.config.max_seq_length |
| 64 | assert len(attention_mask) == self.wrapper.config.max_seq_length |
| 65 | assert len(token_type_ids) == self.wrapper.config.max_seq_length |
| 66 | assert len(block_flag) == self.wrapper.config.max_seq_length |
| 67 | |
| 68 | label = self.label_map[example.label] if example.label is not None else -100 |
| 69 | logits = example.logits if example.logits else [-1] |
| 70 | |
| 71 | if labelled: |
| 72 | mlm_labels = self.pvp.get_mask_positions(input_ids) |
| 73 | else: |
| 74 | mlm_labels = [-1] * self.wrapper.config.max_seq_length |
| 75 | |
| 76 | return InputFeatures(input_ids=input_ids, |
| 77 | attention_mask=attention_mask, |
| 78 | token_type_ids=token_type_ids, |
| 79 | label=label, |
| 80 | mlm_labels=mlm_labels, |
| 81 | logits=logits, |
| 82 | idx=example.idx, |
| 83 | block_flag=block_flag) |
nothing calls this directly
no test coverage detected