| 33 | |
| 34 | |
| 35 | class ScoreMatrixBuilder: |
| 36 | def __init__( |
| 37 | self, |
| 38 | *, |
| 39 | all_games_normalization_scheme: ALL_GAMES_NORMALIZATION_SCHEMES = "none", |
| 40 | score_type: SCORING_TYPES = "per_round_tertiary", |
| 41 | max_round: int = 15, |
| 42 | only_specific_round: bool = False, |
| 43 | include_round_0: bool = False, |
| 44 | ): |
| 45 | """This class builds a win matrix from a log directory, it doesn't fit anything yet. |
| 46 | It also adds a "ALL" game to the win matrix, which is the sum of all games. |
| 47 | There are different choices for normalize the "ALL" game, which is controlled by the all_normalization_scheme parameter. |
| 48 | |
| 49 | The possible values are: |
| 50 | - "none": No normalization, just sum up raw scores |
| 51 | - "by_game_model_pair": Normalize each matchup by its total: wij/(wij+wji)/total_games (NOTE: can't calculate uncertainties for this) |
| 52 | - "by_game": Normalize by total games in each game (NOTE: can't calculate uncertainties for this) |
| 53 | |
| 54 | The `score_type` parameter controls how the score is calculated for each round. The possible values are: |
| 55 | - "per_round_tertiary": Returns 0.0, 0.5, or 1.0 for the score of each player for each round, |
| 56 | depending on the "winner" field in the stats dictionary. |
| 57 | - "per_round_float": The "float" score type returns the scores based on performance over sims |
| 58 | - "per_round_tertiary_p_value": The "tertiary_p_value" score type returns 0.0, 0.5, or 1.0 for the score of each player, |
| 59 | similar to the "tertiary" score type, but if the p-value is greater than 0.05, it concludes |
| 60 | a draw. |
| 61 | - "per_tournament_boolean_drop_draws": The "boolean_drop_draws" score type returns 0.0 or 1.0 for the score of each player, |
| 62 | depending on the "winner" field in the stats dictionary. This is the only score type that gives proper uncertainties for the win matrix. |
| 63 | |
| 64 | The `max_round` parameter controls the maximum number of rounds to include in the score calculation (default: 15). |
| 65 | The `only_specific_round` parameter controls whether to only include the specific round (True) or all rounds up to max_round (False). |
| 66 | The `include_round_0` parameter controls whether round 0 is counted. In normal PvP/climbing |
| 67 | tournaments round 0 is the identical-codebases baseline and is excluded. For ladder |
| 68 | construction (`ladder make`, `tournament.rounds: 0`) round 0 IS the match, so set this True. |
| 69 | """ |
| 70 | self.win_matrix: dict[str, dict[tuple[str, str], list[float]]] = defaultdict( |
| 71 | lambda: defaultdict(lambda: [0.0, 0.0]) |
| 72 | ) |
| 73 | """game name -> (player1, player2) -> [wins, losses]""" |
| 74 | self.all_normalization_scheme = all_games_normalization_scheme |
| 75 | self.score_type = score_type |
| 76 | self.max_round = max_round |
| 77 | self.only_specific_round = only_specific_round |
| 78 | self.include_round_0 = include_round_0 |
| 79 | self._samples: dict[str, dict[tuple[str, str], list[tuple[float, float]]]] = defaultdict( |
| 80 | lambda: defaultdict(list) |
| 81 | ) |
| 82 | |
| 83 | def _get_unique_model_name(self, model: str) -> str: |
| 84 | return model.rpartition("/")[2] |
| 85 | |
| 86 | def _get_sorted_pair(self, p1: str, p2: str) -> tuple[str, str]: |
| 87 | return tuple(sorted([p1, p2])) |
| 88 | |
| 89 | def _get_round_score(self, stats: dict, player_names: list[str], game_name: str) -> tuple[float, float]: |
| 90 | """Calculate score for a round. |
| 91 | |
| 92 | Returns (p1_score, p2_score) where each is 0.0, 0.5, or 1.0. |