Generate a batch
(data_iterator)
| 86 | |
| 87 | |
| 88 | def get_batch(data_iterator): |
| 89 | """Generate a batch""" |
| 90 | args = get_args() |
| 91 | tokenizer = get_tokenizer() |
| 92 | |
| 93 | # Items and their type. |
| 94 | keys = ["input_ids", "attention_mask", "labels"] |
| 95 | datatype = torch.int64 |
| 96 | |
| 97 | # Broadcast data. |
| 98 | if data_iterator is not None: |
| 99 | data = next(data_iterator) |
| 100 | else: |
| 101 | data = None |
| 102 | data_b = mpu.broadcast_data(keys, data, datatype) |
| 103 | |
| 104 | # Unpack. |
| 105 | tokens_ = data_b["input_ids"].contiguous() |
| 106 | # attn_mask_ = data_b["attention_mask"].contiguous() |
| 107 | labels_ = data_b["labels"].contiguous() |
| 108 | |
| 109 | tokens = tokens_[:, :-1] |
| 110 | labels = labels_[:, 1:] |
| 111 | |
| 112 | # Get the masks and postition ids. |
| 113 | attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids( |
| 114 | tokens, |
| 115 | tokenizer.eod, |
| 116 | args.reset_position_ids, |
| 117 | args.reset_attention_mask, |
| 118 | args.eod_mask_loss, |
| 119 | ) |
| 120 | |
| 121 | # mask loss to avoid predicting prompt and paddings |
| 122 | prompt_loss_mask = labels >= 0 |
| 123 | loss_mask = prompt_loss_mask * loss_mask |
| 124 | |
| 125 | return tokens, labels, loss_mask, attention_mask, position_ids |
| 126 | |
| 127 | |
| 128 | def get_batch_pipe(data): |
no test coverage detected