Attach embeddings and normalized ranking features to fetched papers.
(papers: List[Dict])
| 874 | |
| 875 | |
| 876 | def prepare_paper_features(papers: List[Dict]) -> List[Dict]: |
| 877 | """Attach embeddings and normalized ranking features to fetched papers.""" |
| 878 | if not papers: |
| 879 | return papers |
| 880 | |
| 881 | embedding_texts = [build_paper_text(paper) for paper in papers] |
| 882 | embeddings = embed_batch(embedding_texts) |
| 883 | embedding_service = get_embedding_service() |
| 884 | print(f"Embedding descriptor: {embedding_service.descriptor}") |
| 885 | |
| 886 | for paper, embedding in zip(papers, embeddings): |
| 887 | paper["embedding"] = embedding |
| 888 | paper["embedding_model"] = embedding_service.descriptor |
| 889 | paper["institution"] = str(paper.get("institution") or "") |
| 890 | paper["quality_score"] = estimate_quality_score(paper) |
| 891 | |
| 892 | semantic_topics = canonicalize_direction_terms( |
| 893 | extract_topics_from_text(paper.get("title", ""), paper.get("abstract", "")), |
| 894 | keep_unknown=True, |
| 895 | ) |
| 896 | source_categories = list(paper.get("categories", [])) |
| 897 | if paper.get("source") == "openreview": |
| 898 | source_categories.append(paper.get("venue", "conference")) |
| 899 | elif paper.get("source") == "journal": |
| 900 | source_categories.append(paper.get("journal", "journal")) |
| 901 | |
| 902 | expanded_terms = expand_direction_terms(semantic_topics) |
| 903 | paper["keywords"] = dedupe_preserve_order(source_categories + semantic_topics) |
| 904 | paper["topics"] = semantic_topics |
| 905 | paper["direction_terms"] = expanded_terms |
| 906 | |
| 907 | return papers |
| 908 | |
| 909 | |
| 910 | def extract_topics_from_text(title: str, abstract: str = "") -> List[str]: |
no test coverage detected