(text_a_ids, answer_ids, max_seq_length, tokenizer, args=None, add_cls=True,
add_sep=False, add_eos=True, mask_id=None)
| 104 | |
| 105 | |
| 106 | def build_uni_input_from_ids(text_a_ids, answer_ids, max_seq_length, tokenizer, args=None, add_cls=True, |
| 107 | add_sep=False, add_eos=True, mask_id=None): |
| 108 | if mask_id is None: |
| 109 | mask_id = tokenizer.get_command('MASK').Id |
| 110 | eos_id = tokenizer.get_command('eos').Id |
| 111 | cls_id = tokenizer.get_command('ENC').Id |
| 112 | sop_id = tokenizer.get_command('sop').Id |
| 113 | ids = [] |
| 114 | if add_cls: |
| 115 | ids = [cls_id] |
| 116 | ids.append(mask_id) |
| 117 | sep = len(ids) |
| 118 | mask_position = sep - 1 |
| 119 | position_ids = list(range(len(ids))) |
| 120 | block_position_ids = [0] * len(ids) |
| 121 | ids.append(sop_id) |
| 122 | # A |
| 123 | ids.extend(text_a_ids) |
| 124 | target_ids = [0] * (len(ids) - 1) |
| 125 | loss_masks = [0] * (len(ids) - 1) |
| 126 | # Piece |
| 127 | ids.extend(answer_ids[:-1]) |
| 128 | target_ids.extend(answer_ids) |
| 129 | loss_masks.extend([1] * len(answer_ids)) |
| 130 | position_ids.extend([mask_position] * (len(ids) - len(position_ids))) |
| 131 | block_position_ids.extend(range(1, len(ids) - len(block_position_ids) + 1)) |
| 132 | # Padding. |
| 133 | padding_length = max_seq_length - len(ids) |
| 134 | if padding_length > 0: |
| 135 | ids.extend([eos_id] * padding_length) |
| 136 | position_ids.extend([0] * padding_length) |
| 137 | block_position_ids.extend([0] * padding_length) |
| 138 | target_ids.extend([0] * padding_length) |
| 139 | loss_masks.extend([0] * padding_length) |
| 140 | position_ids = [position_ids, block_position_ids] |
| 141 | return ids, None, None, position_ids, sep, target_ids, loss_masks |
| 142 | |
| 143 | |
| 144 | def build_input_from_ids(text_a_ids, text_b_ids, answer_ids, max_seq_length, tokenizer, args=None, add_cls=True, |
no test coverage detected