()
| 124 | |
| 125 | |
| 126 | def run_experiment(): |
| 127 | train_cases = load_jsonl("bench/data/train.jsonl") |
| 128 | test_cases = load_jsonl("bench/data/test.jsonl") |
| 129 | |
| 130 | print(f"Train: {len(train_cases)}, Test: {len(test_cases)}") |
| 131 | |
| 132 | # Extract handcrafted features |
| 133 | print("Extracting handcrafted features...") |
| 134 | t0 = time.time() |
| 135 | X_train_hc = np.array([extract_handcrafted_features(c["prompt"], c.get("system_prompt")) for c in train_cases]) |
| 136 | X_test_hc = np.array([extract_handcrafted_features(c["prompt"], c.get("system_prompt")) for c in test_cases]) |
| 137 | y_train = np.array([TIER_IDX[c["expected_tier"]] for c in train_cases]) |
| 138 | y_test = np.array([TIER_IDX[c["expected_tier"]] for c in test_cases]) |
| 139 | hc_time = time.time() - t0 |
| 140 | print(f" Handcrafted features: {X_train_hc.shape[1]} dims, {hc_time:.1f}s") |
| 141 | |
| 142 | # Extract embeddings |
| 143 | print("Loading embedding model (MiniLM-L6-v2)...") |
| 144 | t0 = time.time() |
| 145 | emb_model = SentenceTransformer("all-MiniLM-L6-v2") |
| 146 | load_time = time.time() - t0 |
| 147 | print(f" Model loaded in {load_time:.1f}s") |
| 148 | |
| 149 | print("Generating embeddings...") |
| 150 | t0 = time.time() |
| 151 | train_prompts = [c["prompt"] for c in train_cases] |
| 152 | test_prompts = [c["prompt"] for c in test_cases] |
| 153 | X_train_emb = emb_model.encode(train_prompts, show_progress_bar=False, normalize_embeddings=True) |
| 154 | X_test_emb = emb_model.encode(test_prompts, show_progress_bar=False, normalize_embeddings=True) |
| 155 | emb_time = time.time() - t0 |
| 156 | print(f" Embeddings: {X_train_emb.shape[1]} dims, {emb_time:.1f}s") |
| 157 | |
| 158 | # Combined features |
| 159 | X_train_combined = np.hstack([X_train_hc, X_train_emb]) |
| 160 | X_test_combined = np.hstack([X_test_hc, X_test_emb]) |
| 161 | |
| 162 | # Normalize for MLP stability |
| 163 | mean_hc = X_train_hc.mean(axis=0) |
| 164 | std_hc = X_train_hc.std(axis=0) + 1e-8 |
| 165 | X_train_hc_norm = (X_train_hc - mean_hc) / std_hc |
| 166 | X_test_hc_norm = (X_test_hc - mean_hc) / std_hc |
| 167 | |
| 168 | mean_comb = X_train_combined.mean(axis=0) |
| 169 | std_comb = X_train_combined.std(axis=0) + 1e-8 |
| 170 | X_train_comb_norm = (X_train_combined - mean_comb) / std_comb |
| 171 | X_test_comb_norm = (X_test_combined - mean_comb) / std_comb |
| 172 | |
| 173 | mean_emb = X_train_emb.mean(axis=0) |
| 174 | std_emb = X_train_emb.std(axis=0) + 1e-8 |
| 175 | X_train_emb_norm = (X_train_emb - mean_emb) / std_emb |
| 176 | X_test_emb_norm = (X_test_emb - mean_emb) / std_emb |
| 177 | |
| 178 | # ─── Run experiments ─── |
| 179 | configs = [ |
| 180 | ("Perceptron + handcrafted (current)", AvgPerceptron, X_train_hc, X_test_hc, {"epochs": 12}), |
| 181 | ("Perceptron + embedding only", AvgPerceptron, X_train_emb, X_test_emb, {"epochs": 12}), |
| 182 | ("Perceptron + handcrafted + embedding", AvgPerceptron, X_train_combined, X_test_combined, {"epochs": 12}), |
| 183 | ("MLP + handcrafted", SimpleMLP, X_train_hc_norm, X_test_hc_norm, {"epochs": 60, "hidden": 64}), |
no test coverage detected