| 151 | |
| 152 | # Check the model accuracy on the validation dataset |
| 153 | def validate(model, valid_dl, loss_fn, special_symbols): |
| 154 | |
| 155 | # Object for accumulating losses |
| 156 | losses = 0 |
| 157 | |
| 158 | # Turn off gradients a moment |
| 159 | model.eval() |
| 160 | |
| 161 | for src, tgt in tqdm(valid_dl): |
| 162 | |
| 163 | src = src.to(DEVICE) |
| 164 | tgt = tgt.to(DEVICE) |
| 165 | |
| 166 | # We need to reshape the input slightly to fit into the transformer |
| 167 | tgt_input = tgt[:-1, :] |
| 168 | |
| 169 | # Create masks |
| 170 | src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input, special_symbols["<pad>"], DEVICE) |
| 171 | |
| 172 | # Pass into model, get probability over the vocab out |
| 173 | logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask) |
| 174 | |
| 175 | # Get original shape back, compute loss, accumulate that loss |
| 176 | tgt_out = tgt[1:, :] |
| 177 | loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1)) |
| 178 | losses += loss.item() |
| 179 | |
| 180 | # Return the average loss |
| 181 | return losses / len(list(valid_dl)) |
| 182 | |
| 183 | # Train the model |
| 184 | def main(opts): |