Load a JSONL split — returns embeddings, metadata features, and labels.
(path: Path, model)
| 28 | |
| 29 | |
| 30 | def load_split(path: Path, model): |
| 31 | """Load a JSONL split — returns embeddings, metadata features, and labels.""" |
| 32 | from uncommon_route.signals.embedding import _extract_last_user_message, EmbeddingSignal |
| 33 | |
| 34 | rows = [] |
| 35 | with open(path) as f: |
| 36 | for line in f: |
| 37 | line = line.strip() |
| 38 | if line: |
| 39 | rows.append(json.loads(line)) |
| 40 | |
| 41 | texts = [] |
| 42 | meta_feats = [] |
| 43 | labels = [] |
| 44 | for row in rows: |
| 45 | messages = row.get("messages", []) |
| 46 | text = _extract_last_user_message(messages) |
| 47 | if text.strip(): |
| 48 | texts.append(text) |
| 49 | meta_feats.append(EmbeddingSignal._extract_meta_features(messages, text)) |
| 50 | labels.append(row["target_tier_id"]) |
| 51 | |
| 52 | embeddings = model.encode(texts, normalize_embeddings=True, show_progress_bar=False) |
| 53 | return ( |
| 54 | np.array(embeddings, dtype=np.float32), |
| 55 | np.array(meta_feats, dtype=np.float32), |
| 56 | np.array(labels, dtype=np.int32), |
| 57 | ) |
| 58 | |
| 59 | |
| 60 | def main(): |
no test coverage detected