(self, idx)
| 447 | return len(self.example_list) |
| 448 | |
| 449 | def __getitem__(self, idx): |
| 450 | example = self.example_list[idx] |
| 451 | cls_id = self.tokenizer.get_command('ENC').Id |
| 452 | mask_token = 'sMASK' if self.args.task_mask else 'MASK' |
| 453 | mask_id = self.tokenizer.get_command(mask_token).Id |
| 454 | pad_id = self.tokenizer.get_command('pad').Id |
| 455 | sop_id = self.tokenizer.get_command('sop').Id |
| 456 | eop_id = self.tokenizer.get_command('eop').Id |
| 457 | if self.task in ["gigaword", "cnn_dm", "cnn_dm_original", "xsum"]: |
| 458 | source_text, target_text = example.text_a, example.text_b |
| 459 | source_tokens = self.tokenizer.EncodeAsIds(" " + source_text).tokenization |
| 460 | prompt = [cls_id, mask_id] + self.tokenizer.EncodeAsIds(" Content:").tokenization |
| 461 | if len(source_tokens) > self.max_src_length - len(prompt): |
| 462 | source_tokens = source_tokens[:self.max_src_length - len(prompt)] |
| 463 | source_tokens = prompt + source_tokens |
| 464 | elif self.task == "squad_generation": |
| 465 | source_text = example.text_a |
| 466 | target_text, answer = example.meta["question"], example.meta["answer"] |
| 467 | source_tokens = self.tokenizer.EncodeAsIds(source_text.rstrip() + " Question:").tokenization |
| 468 | answer_tokens = self.tokenizer.EncodeAsIds(" Answer: " + answer).tokenization |
| 469 | if len(source_tokens) > self.max_src_length - len(answer_tokens) - 2: |
| 470 | max_src_length = self.max_src_length - len(answer_tokens) - 2 |
| 471 | answer_pattern = self.tokenizer.EncodeAsIds(" " + answer).tokenization |
| 472 | |
| 473 | def sub_finder(mylist, pattern): |
| 474 | matches = [] |
| 475 | for i in range(len(mylist)): |
| 476 | if mylist[i] == pattern[0] and mylist[i:i + len(pattern)] == pattern: |
| 477 | matches.append(i) |
| 478 | return matches |
| 479 | |
| 480 | answer_indices = sub_finder(source_tokens, answer_pattern) |
| 481 | if len(answer_indices) == 0: |
| 482 | print(f"Answer {answer} not exists in the source text") |
| 483 | source_tokens = source_tokens[:max_src_length] |
| 484 | else: |
| 485 | start_index = max(answer_indices[0] - max_src_length // 2, 0) |
| 486 | source_tokens = source_tokens[start_index: start_index + max_src_length] |
| 487 | source_tokens = [cls_id] + source_tokens + [mask_id] + answer_tokens |
| 488 | elif self.task in ["squad", "squad_v1"]: |
| 489 | source_text = example.text_a |
| 490 | target_text = example.meta["answer"].strip() |
| 491 | question = example.meta["question"].strip() |
| 492 | source_tokens = self.tokenizer.EncodeAsIds(" " + source_text.rstrip()).tokenization |
| 493 | question_tokens = self.tokenizer.EncodeAsIds(" " + question).tokenization |
| 494 | period_id = self.tokenizer.TokenToId('.') |
| 495 | max_src_length = self.max_src_length - len(question_tokens) - 3 |
| 496 | if max_src_length <= 0: |
| 497 | print(question) |
| 498 | assert max_src_length > 0 |
| 499 | source_tokens = [cls_id] + question_tokens + [mask_id, period_id] + source_tokens[:max_src_length] |
| 500 | elif self.task in ["cmrc"]: |
| 501 | mask_id = self.tokenizer.get_command('MASK').Id |
| 502 | source_text = example.text_a |
| 503 | target_text = example.meta["answer"].strip() |
| 504 | question = example.meta["question"].strip() |
| 505 | source_tokens = self.tokenizer.EncodeAsIds(source_text.rstrip()).tokenization |
| 506 | question_tokens = self.tokenizer.EncodeAsIds("问题：" + question + "答案：").tokenization |
nothing calls this directly
no test coverage detected