(enc_ids, answer_ids, max_seq_length, max_dec_seq_length, tokenizer)
| 231 | |
| 232 | |
| 233 | def build_decoder_input(enc_ids, answer_ids, max_seq_length, max_dec_seq_length, tokenizer): |
| 234 | mask_id = tokenizer.get_command('MASK').Id |
| 235 | eos_id = tokenizer.get_command('eos').Id |
| 236 | sop_id = tokenizer.get_command('sop').Id |
| 237 | enc_len = len(enc_ids) |
| 238 | masks = [] |
| 239 | # TODO: it probably takes too much memory |
| 240 | # for i in range(max_dec_seq_length): |
| 241 | # m = [1]*enc_len + [0]*(max_seq_length - enc_len) + [1]*(i+1) + [0]*(max_dec_seq_length-1-i) |
| 242 | # masks.append(m) |
| 243 | mask_position = enc_ids.index(mask_id) |
| 244 | len_answer = len(answer_ids) |
| 245 | ids = [sop_id] + answer_ids[:-1] |
| 246 | types = [0] * len_answer # not used |
| 247 | paddings = [1] * len_answer |
| 248 | position_ids = [mask_position] * len_answer |
| 249 | block_position_ids = list(range(1, len_answer + 1)) |
| 250 | target_ids = answer_ids |
| 251 | loss_masks = [1] * len_answer |
| 252 | # Padding. |
| 253 | padding_length = max_dec_seq_length - len(ids) |
| 254 | if padding_length > 0: |
| 255 | ids.extend([eos_id] * padding_length) |
| 256 | types.extend([0] * padding_length) |
| 257 | paddings.extend([0] * padding_length) |
| 258 | position_ids.extend([0] * padding_length) |
| 259 | block_position_ids.extend([0] * padding_length) |
| 260 | target_ids.extend([0] * padding_length) |
| 261 | loss_masks.extend([0] * padding_length) |
| 262 | position_ids = [position_ids, block_position_ids] |
| 263 | return ids, types, paddings, position_ids, masks, target_ids, loss_masks |
| 264 | |
| 265 | |
| 266 | def build_sample(ids, types=None, paddings=None, positions=None, masks=None, label=None, unique_id=None, target=None, |
no test coverage detected