(self, example: InputExample, tokenizer, seq_length, args)
| 266 | return example.text_a, example.text_b |
| 267 | |
| 268 | def encode(self, example: InputExample, tokenizer, seq_length, args): |
| 269 | text_a, text_b = self.get_classifier_input(example, tokenizer) |
| 270 | tokens_a = tokenizer.EncodeAsIds(text_a).tokenization |
| 271 | tokens_b = tokenizer.EncodeAsIds(text_b).tokenization |
| 272 | num_special_tokens = num_special_tokens_to_add(tokens_a, tokens_b, None, add_cls=True, add_sep=True, |
| 273 | add_piece=False) |
| 274 | if len(tokens_a) + len(tokens_b) + num_special_tokens > seq_length: |
| 275 | self.num_truncated += 1 |
| 276 | data = build_input_from_ids(tokens_a, tokens_b, None, seq_length, tokenizer, args=args, |
| 277 | add_cls=True, add_sep=True, add_piece=False) |
| 278 | ids, types, paddings, position_ids, sep, target_ids, loss_masks = data |
| 279 | label = 0 |
| 280 | if example.label is not None: |
| 281 | label = example.label |
| 282 | label = self.get_labels().index(label) |
| 283 | if args.pretrained_bert: |
| 284 | sample = build_sample(ids, label=label, types=types, paddings=paddings, |
| 285 | unique_id=example.guid) |
| 286 | else: |
| 287 | sample = build_sample(ids, positions=position_ids, masks=sep, label=label, |
| 288 | unique_id=example.guid) |
| 289 | return sample |
| 290 | |
| 291 | |
| 292 | class SuperGLUEProcessor(DataProcessor): |
no test coverage detected