(model, dataloader, valid_dataloader, test_dataloader, args)
| 56 | |
| 57 | |
| 58 | def train_model(model, dataloader, valid_dataloader, test_dataloader, args): |
| 59 | model.to(args.device) |
| 60 | print(args.device) |
| 61 | optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) |
| 62 | total_training_steps = len(dataloader) * args.num_epochs |
| 63 | scheduler = get_linear_schedule_with_warmup( |
| 64 | optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=total_training_steps |
| 65 | ) |
| 66 | |
| 67 | criterion = torch.nn.CrossEntropyLoss(label_smoothing=0.1) |
| 68 | |
| 69 | use_bf16 = args.precision == "bf16" |
| 70 | scaler = torch.cuda.amp.GradScaler(enabled=not use_bf16) |
| 71 | autocast_dtype = torch.bfloat16 if use_bf16 else torch.float16 |
| 72 | |
| 73 | model.train() |
| 74 | os.makedirs(args.checkpoint_dir, exist_ok=True) |
| 75 | start_time = time.time() |
| 76 | for epoch in range(args.num_epochs): |
| 77 | total_loss = 0.0 |
| 78 | progress_bar = tqdm(dataloader, desc=f"Epoch {epoch + 1}/{args.num_epochs}", unit="batch") |
| 79 | |
| 80 | for input_ids, target_tokens, attention_mask, input_lengths in progress_bar: |
| 81 | input_ids = input_ids.to(args.device) |
| 82 | target_tokens = target_tokens.to(args.device) |
| 83 | attention_mask = attention_mask.to(args.device) |
| 84 | current_recurrence = args.recurrence |
| 85 | model.num_iterations = current_recurrence |
| 86 | optimizer.zero_grad() |
| 87 | |
| 88 | with torch.cuda.amp.autocast(dtype=autocast_dtype, enabled=True): |
| 89 | outputs = model(input_ids=input_ids, attention_mask=attention_mask) |
| 90 | logits = outputs.logits |
| 91 | |
| 92 | final_logits = logits[:, -1, :] |
| 93 | |
| 94 | loss = criterion(final_logits, target_tokens) |
| 95 | |
| 96 | if use_bf16: |
| 97 | loss.backward() |
| 98 | optimizer.step() |
| 99 | else: |
| 100 | scaler.scale(loss).backward() |
| 101 | scaler.step(optimizer) |
| 102 | scaler.update() |
| 103 | |
| 104 | scheduler.step() |
| 105 | total_loss += loss.item() |
| 106 | progress_bar.set_postfix(loss=loss.item(), lr=optimizer.param_groups[0]['lr']) |
| 107 | |
| 108 | avg_loss = total_loss / len(dataloader) |
| 109 | print(f"Epoch {epoch + 1}/{args.num_epochs}, Loss: {avg_loss:.4f}") |
| 110 | |
| 111 | print("Running Evaluation...") |
| 112 | |
| 113 | model.num_iterations = args.recurrence |
| 114 | test_acc_per_type = evaluate_model_test(model, test_dataloader, args.device) # Get per-type accuracy |
| 115 | test_acc_overall = sum(test_acc_per_type.values()) / len(test_acc_per_type) if test_acc_per_type else 0.0 |
no test coverage detected