(log_dir: Path)
| 23 | |
| 24 | |
| 25 | def main(log_dir: Path): |
| 26 | # Assuming directory structure is: |
| 27 | # logs/<user_id>/<game_id> |
| 28 | # - players/ |
| 29 | # - game.log |
| 30 | # - metadata.json |
| 31 | model_profiles = {} |
| 32 | for game_log_folder in tqdm([x.parent for x in log_dir.rglob("metadata.json")]): |
| 33 | game_id = game_log_folder.name.split(".")[1] |
| 34 | player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()] |
| 35 | metadata = json.load(open(game_log_folder / "metadata.json")) |
| 36 | try: |
| 37 | player_to_model = { |
| 38 | x["name"]: x["config"]["model"]["model_name"].strip("@").split("/")[-1] |
| 39 | for x in metadata["config"]["players"] |
| 40 | } |
| 41 | except KeyError: |
| 42 | continue |
| 43 | num_rounds = len(metadata["round_stats"]) |
| 44 | |
| 45 | # Only count each unique model once per game |
| 46 | unique_models = {player_to_model[player] for player in player_ids} |
| 47 | for model_name in unique_models: |
| 48 | k = f"{game_id}.{model_name}" |
| 49 | if k in model_profiles: |
| 50 | model_profiles[k].count += num_rounds |
| 51 | else: |
| 52 | # Use the first player_id that matches this model_name for display |
| 53 | player_id = next(pid for pid in player_ids if player_to_model[pid] == model_name) |
| 54 | model_profiles[k] = PlayerGameProfile( |
| 55 | player_id=player_id, model_name=model_name, game_id=game_id, count=num_rounds |
| 56 | ) |
| 57 | |
| 58 | for round, details in metadata["round_stats"].items(): |
| 59 | if round == "0": |
| 60 | # Skip initial round |
| 61 | continue |
| 62 | winner = details["winner"] |
| 63 | if winner != RESULT_TIE: |
| 64 | model_profiles[f"{game_id}.{player_to_model[winner]}"].wins += 1 |
| 65 | |
| 66 | print("Player profiles:") |
| 67 | lines = [ |
| 68 | f" - {profile.model_name} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})" |
| 69 | for profile in model_profiles.values() |
| 70 | ] |
| 71 | print("\n".join(sorted(lines))) |
| 72 | |
| 73 | # Player-specific (game-agnostic) win rates (micro average) |
| 74 | total_wins = {} |
| 75 | total_games = {} |
| 76 | model_names = {} |
| 77 | for profile in model_profiles.values(): |
| 78 | mid = profile.model_name |
| 79 | total_wins[mid] = total_wins.get(mid, 0) + profile.wins |
| 80 | total_games[mid] = total_games.get(mid, 0) + profile.count |
| 81 | model_names[mid] = profile.model_name |
| 82 |
no test coverage detected