| 1001 | self._create_rank_matrix_plot(players, rank_samples, bootstrap_dir) |
| 1002 | self._create_elo_violin_plot(players, elo_samples, baseline_elos, bootstrap_dir) |
| 1003 | |
| 1004 | return { |
| 1005 | "kendall_tau": mean_tau, |
| 1006 | "spearman_rho": mean_rho, |
| 1007 | "footrule": mean_foot, |
| 1008 | "top1_consistency": top1_consistency, |
| 1009 | "pairwise_agreement": pairwise_agreement, |
| 1010 | "topk_overlap": {k: float(np.mean(topk_overlap[k])) for k in topks}, |
| 1011 | } |
| 1012 | |
| 1013 | |
| 1014 | class EloVsMaxRounds: |
| 1015 | def __init__( |
| 1016 | self, |
| 1017 | *, |
| 1018 | log_dir: Path, |
| 1019 | max_rounds: int = 15, |
| 1020 | all_games_normalization_scheme: ALL_GAMES_NORMALIZATION_SCHEMES = "none", |
| 1021 | score_type: SCORING_TYPES = "per_round_tertiary", |
| 1022 | regularization: float = 0.01, |
| 1023 | output_dir: Path | None = None, |
| 1024 | games: list[str] | None = None, |
| 1025 | ): |
| 1026 | self.log_dir = log_dir |
| 1027 | self.max_rounds = max_rounds |
| 1028 | self.all_games_normalization_scheme = all_games_normalization_scheme |
| 1029 | self.score_type = score_type |
| 1030 | self.regularization = regularization |
| 1031 | self.output_dir = output_dir |
| 1032 | self.games = games |
| 1033 | |
| 1034 | @staticmethod |
| 1035 | def _save_plot(output_dir: Path, filename_base: str) -> None: |
| 1036 | """Save plot in both PDF and PNG formats.""" |
| 1037 | for fmt in ["pdf", "png"]: |
| 1038 | output_path = output_dir / f"{filename_base}.{fmt}" |
| 1039 | plt.savefig(output_path, format=fmt, bbox_inches="tight", dpi=300 if fmt == "png" else None) |
| 1040 | logger.info(f"Saved plot: {output_path}") |
| 1041 | |
| 1042 | def run(self) -> None: |
| 1043 | """Calculate Elo for max rounds from 1 to max_rounds and plot the results.""" |
| 1044 | logger.info(f"Calculating Elo for max rounds 1 to {self.max_rounds}") |
| 1045 | |
| 1046 | # Dictionary to store results: {max_round: {game_name: {player: elo}}} |
| 1047 | results_by_max_round: dict[int, dict[str, dict[str, float]]] = {} |
| 1048 | |
| 1049 | for max_round in tqdm(range(1, self.max_rounds + 1), desc="Processing max rounds"): |
| 1050 | builder = ScoreMatrixBuilder( |
| 1051 | all_games_normalization_scheme=self.all_games_normalization_scheme, |
| 1052 | score_type=self.score_type, |
| 1053 | max_round=max_round, |
| 1054 | ) |
| 1055 | builder.build(self.log_dir) |
| 1056 | |
| 1057 | results_by_max_round[max_round] = {} |
| 1058 | |
| 1059 | for game_name, matchups in builder.win_matrix.items(): |
| 1060 | if self.games is not None and game_name not in self.games: |