Constrain reading-report recommendation labels to ranking/evaluation evidence.
(paper: Dict[str, Any], proposed_label: Any, analysis_source: Any = None)
| 866 | |
| 867 | def calibrate_recommendation_label(paper: Dict[str, Any], proposed_label: Any, analysis_source: Any = None) -> str: |
| 868 | """Constrain reading-report recommendation labels to ranking/evaluation evidence.""" |
| 869 | system_label = _clean_text( |
| 870 | paper.get("system_label") |
| 871 | or paper.get("relevance_level") |
| 872 | or paper.get("category") |
| 873 | ) |
| 874 | oracle_label = _clean_text(paper.get("oracle_label")) |
| 875 | analysis_source_text = _clean_text(analysis_source).lower() |
| 876 | |
| 877 | try: |
| 878 | system_score = float(paper.get("system_score", paper.get("relevance_score", 0.0)) or 0.0) |
| 879 | except (TypeError, ValueError): |
| 880 | system_score = 0.0 |
| 881 | for score_key in ("score", "relevance", "ranking_score", "final_score"): |
| 882 | try: |
| 883 | candidate_score = float(paper.get(score_key, 0.0) or 0.0) |
| 884 | except (TypeError, ValueError): |
| 885 | candidate_score = 0.0 |
| 886 | system_score = max(system_score, candidate_score) |
| 887 | try: |
| 888 | oracle_score = float(paper.get("oracle_score", 0.0) or 0.0) |
| 889 | except (TypeError, ValueError): |
| 890 | oracle_score = 0.0 |
| 891 | |
| 892 | if system_label == "must_read" or oracle_label == "strong_relevant" or system_score >= 0.82 or oracle_score >= 0.72: |
| 893 | max_rank = 3 |
| 894 | default_rank = 3 |
| 895 | elif system_label == "high_relevant" or oracle_label == "relevant" or system_score >= 0.62 or oracle_score >= 0.48: |
| 896 | max_rank = 2 |
| 897 | default_rank = 2 |
| 898 | elif system_label == "maybe_interested" or oracle_label == "weak_relevant" or system_score >= 0.36 or oracle_score >= 0.25: |
| 899 | max_rank = 1 |
| 900 | default_rank = 1 |
| 901 | else: |
| 902 | max_rank = 0 |
| 903 | default_rank = 0 |
| 904 | |
| 905 | is_hard_priority = system_label == "must_read" or oracle_label == "strong_relevant" or oracle_score >= 0.72 |
| 906 | if not is_hard_priority and analysis_source_text not in {"pdf", "source_page"} and max_rank > 2: |
| 907 | max_rank = 2 |
| 908 | |
| 909 | proposed_rank = _label_rank(str(proposed_label or "")) |
| 910 | if not _clean_text(proposed_label): |
| 911 | proposed_rank = default_rank |
| 912 | |
| 913 | return _rank_label(min(max(proposed_rank, default_rank), max_rank)) |
| 914 | |
| 915 | |
| 916 | def build_recommendation_calibration_metadata( |
| 917 | paper: Dict[str, Any], |
no test coverage detected