()
| 58 | |
| 59 | |
| 60 | def main(): |
| 61 | splits_dir = Path("uncommon_route/data/v2_splits") |
| 62 | |
| 63 | # Check all splits exist |
| 64 | for name in ("train.jsonl", "calibration.jsonl", "holdout.jsonl"): |
| 65 | if not (splits_dir / name).exists(): |
| 66 | print(f"ERROR: {splits_dir / name} not found. Run split_data.py first.") |
| 67 | sys.exit(1) |
| 68 | |
| 69 | print("Loading embedding model (frozen, no fine-tuning)...") |
| 70 | from sentence_transformers import SentenceTransformer |
| 71 | model = SentenceTransformer("BAAI/bge-small-en-v1.5") |
| 72 | |
| 73 | print("Computing embeddings + metadata features for each split...") |
| 74 | E_train, M_train, y_train = load_split(splits_dir / "train.jsonl", model) |
| 75 | E_cal, M_cal, y_cal = load_split(splits_dir / "calibration.jsonl", model) |
| 76 | E_holdout, M_holdout, y_holdout = load_split(splits_dir / "holdout.jsonl", model) |
| 77 | |
| 78 | print(f" Train: {len(y_train)} samples ({E_train.shape[1]}d emb + {M_train.shape[1]} meta)") |
| 79 | print(f" Calibration: {len(y_cal)} samples") |
| 80 | print(f" Holdout: {len(y_holdout)} samples") |
| 81 | |
| 82 | # Scale metadata features |
| 83 | from sklearn.preprocessing import StandardScaler |
| 84 | scaler = StandardScaler() |
| 85 | M_train_s = scaler.fit_transform(M_train) |
| 86 | M_cal_s = scaler.transform(M_cal) |
| 87 | M_holdout_s = scaler.transform(M_holdout) |
| 88 | |
| 89 | # Combine embedding + scaled metadata |
| 90 | X_train = np.hstack([E_train, M_train_s]) |
| 91 | X_cal = np.hstack([E_cal, M_cal_s]) |
| 92 | X_holdout = np.hstack([E_holdout, M_holdout_s]) |
| 93 | |
| 94 | # Train L2-regularized logistic regression with cross-validation for C |
| 95 | from sklearn.linear_model import LogisticRegressionCV |
| 96 | |
| 97 | print("\nTraining logistic regression on embedding+metadata features...") |
| 98 | clf = LogisticRegressionCV( |
| 99 | Cs=[0.01, 0.1, 0.5, 1.0, 5.0, 10.0], |
| 100 | cv=5, |
| 101 | penalty="l2", |
| 102 | solver="lbfgs", |
| 103 | max_iter=2000, |
| 104 | random_state=42, |
| 105 | ) |
| 106 | clf.fit(X_train, y_train) |
| 107 | |
| 108 | best_C = clf.C_[0] |
| 109 | print(f" Best C: {best_C}") |
| 110 | |
| 111 | # Evaluate on all splits |
| 112 | train_acc = clf.score(X_train, y_train) |
| 113 | cal_acc = clf.score(X_cal, y_cal) |
| 114 | holdout_acc = clf.score(X_holdout, y_holdout) |
| 115 | |
| 116 | print(f"\n{'='*60}") |
| 117 | print(f" OVERFITTING CHECK") |
no test coverage detected