Compute pairwise similarity between a model's solutions at a specific round across multiple games. Use this for both questions 1a (early rounds) and 1b (final round).
(
model: str, opponent: str, arena: str, round_num: int, n_workers: int = 4, similarity: str = "difflib"
)
| 122 | |
| 123 | |
| 124 | def compute_round_consistency( |
| 125 | model: str, opponent: str, arena: str, round_num: int, n_workers: int = 4, similarity: str = "difflib" |
| 126 | ) -> tuple[np.ndarray, np.ndarray]: |
| 127 | """ |
| 128 | Compute pairwise similarity between a model's solutions at a specific round across multiple games. |
| 129 | Use this for both questions 1a (early rounds) and 1b (final round). |
| 130 | """ |
| 131 | folders = get_model_arena_logs([model, opponent], arena) |
| 132 | patches = get_submission_diffs_at_round(folders, model, round_num) |
| 133 | print(f"Found {len(patches)} patches for {model} vs {opponent} in {arena} at round {round_num}") |
| 134 | |
| 135 | # Compute similarity matrix in parallel |
| 136 | patch_list = list(patches.values()) |
| 137 | n = len(patch_list) |
| 138 | similarity_matrix = np.zeros((n, n)) |
| 139 | |
| 140 | with ProcessPoolExecutor(max_workers=n_workers) as executor: |
| 141 | tasks = [(i, patch_list[i], patch_list, similarity) for i in range(n)] |
| 142 | futures = {executor.submit(_compute_similarity_row, task): task for task in tasks} |
| 143 | |
| 144 | for future in tqdm(as_completed(futures), total=n, desc="Computing similarities"): |
| 145 | i, row = future.result() |
| 146 | similarity_matrix[i, :] = row |
| 147 | |
| 148 | # Extract upper triangle for statistics |
| 149 | upper_triangle = similarity_matrix[np.triu_indices(n, k=1)] |
| 150 | |
| 151 | return similarity_matrix, upper_triangle |
| 152 | |
| 153 | |
| 154 | def tag_to_str(tag: dict) -> str: |
no test coverage detected