| 269 | |
| 270 | |
| 271 | def evaluate(model, dataloader, device): |
| 272 | model.eval() |
| 273 | total_loss = 0 |
| 274 | total_tokens = 0 |
| 275 | correct_predictions = 0 |
| 276 | |
| 277 | with torch.no_grad(): |
| 278 | for batch in tqdm(dataloader, desc="Evaluating"): |
| 279 | input_ids = batch["input_ids"].to(device) |
| 280 | attention_mask = batch["attention_mask"].to(device) |
| 281 | labels = batch["labels"].to(device) |
| 282 | |
| 283 | outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels) |
| 284 | loss = outputs.loss |
| 285 | logits = outputs.logits # [B, L, num_labels] |
| 286 | |
| 287 | total_loss += loss.item() |
| 288 | |
| 289 | # Calculate accuracy (only consider labels that are not -100) |
| 290 | active_logits = logits.view(-1, model.config.num_labels) # [B*L, num_labels] |
| 291 | active_labels = labels.view(-1) # [B*L] |
| 292 | |
| 293 | # Filter out valid labels that are not -100 |
| 294 | mask = active_labels != -100 |
| 295 | active_logits = active_logits[mask] |
| 296 | active_labels = active_labels[mask] |
| 297 | |
| 298 | if active_labels.numel() > 0: # Ensure there are valid labels for calculation |
| 299 | predicted_labels = torch.argmax(active_logits, dim=-1) |
| 300 | correct_predictions += (predicted_labels == active_labels).sum().item() |
| 301 | total_tokens += active_labels.numel() |
| 302 | |
| 303 | avg_loss = total_loss / len(dataloader) |
| 304 | accuracy = correct_predictions / total_tokens if total_tokens > 0 else 0.0 |
| 305 | return avg_loss, accuracy |
| 306 | |
| 307 | |
| 308 | # CUDA_VISIBLE_DEVICES=0 python train_predictor.py |