(log_dir: str)
| 101 | |
| 102 | |
| 103 | def main(log_dir: str): |
| 104 | profiles = {} |
| 105 | tournaments = [x.parent for x in log_dir.rglob("metadata.json")] |
| 106 | for game_log_folder in tqdm(tournaments): |
| 107 | with open(game_log_folder / "metadata.json") as f: |
| 108 | metadata = json.load(f) |
| 109 | try: |
| 110 | p2m = { |
| 111 | x["name"]: x["config"]["model"]["model_name"].strip("@").split("/")[-1] |
| 112 | for x in metadata["config"]["players"] |
| 113 | } |
| 114 | except KeyError: |
| 115 | continue |
| 116 | |
| 117 | for model in p2m.values(): |
| 118 | if model not in profiles: |
| 119 | profiles[model] = ModelProfile(name=model) |
| 120 | profiles[model].tournaments.append(game_log_folder.stem) |
| 121 | |
| 122 | for name in p2m.keys(): |
| 123 | traj_files = (game_log_folder / "players" / name).rglob("*.traj.json") |
| 124 | for traj_file in traj_files: |
| 125 | try: |
| 126 | analyzer = TrajectoryAnalyzer(traj_file) |
| 127 | except (json.JSONDecodeError, KeyError, FileNotFoundError): |
| 128 | continue |
| 129 | profiles[p2m[name]].steps.append(analyzer.steps) |
| 130 | |
| 131 | failure_stats = analyzer.failure_stats |
| 132 | profiles[p2m[name]].failed_commands += failure_stats["failed_commands"] |
| 133 | for k, v in failure_stats["failed_command_types"].items(): |
| 134 | profiles[p2m[name]].failed_command_types[k] = profiles[p2m[name]].failed_command_types.get(k, 0) + v |
| 135 | |
| 136 | sep = "=" * 40 |
| 137 | print(f"Models found: {len(profiles)}") |
| 138 | print(f"Tournaments found: {len(tournaments)}") |
| 139 | print(sep) |
| 140 | for profile in profiles.values(): |
| 141 | print(profile) |
| 142 | print(sep) |
| 143 | |
| 144 | |
| 145 | if __name__ == "__main__": |
no test coverage detected