Run code evolution analyses.
(
data_cache: Path = Path("assets/code_evolve_cache_BattleSnake_difflib.jsonl"),
arena: str = "BattleSnake",
similarity: str = "difflib",
)
| 156 | |
| 157 | |
| 158 | def collect_data( |
| 159 | data_cache: Path = Path("assets/code_evolve_cache_BattleSnake_difflib.jsonl"), |
| 160 | arena: str = "BattleSnake", |
| 161 | similarity: str = "difflib", |
| 162 | ): |
| 163 | """Run code evolution analyses.""" |
| 164 | mode, to_skip = "w", [] |
| 165 | if data_cache.exists(): |
| 166 | mode = "a" |
| 167 | with open(data_cache) as f: |
| 168 | for line in f: |
| 169 | entry = json.loads(line) |
| 170 | to_skip.append( |
| 171 | tag_to_str( |
| 172 | { |
| 173 | "model_a": entry["model_a"], |
| 174 | "model_b": entry["model_b"], |
| 175 | "arena": entry["arena"], |
| 176 | "round": entry["round"], |
| 177 | } |
| 178 | ) |
| 179 | ) |
| 180 | print(f"Found cache file, skipping {len(to_skip)} entries.") |
| 181 | |
| 182 | with open(MODELS_PATH) as f: |
| 183 | models = [x["model_name"].rsplit("/")[-1] for x in yaml.safe_load(f)] |
| 184 | |
| 185 | with open(data_cache, mode) as f: |
| 186 | for round in TARGET_ROUNDS: |
| 187 | for i in range(0, len(models)): |
| 188 | for j in range(0, len(models)): |
| 189 | if i == j: |
| 190 | continue |
| 191 | if ( |
| 192 | tag_to_str( |
| 193 | { |
| 194 | "model_a": models[i], |
| 195 | "model_b": models[j], |
| 196 | "arena": arena, |
| 197 | "round": round, |
| 198 | } |
| 199 | ) |
| 200 | in to_skip |
| 201 | ): |
| 202 | continue |
| 203 | try: |
| 204 | sim_matrix, _ = compute_round_consistency( |
| 205 | models[i], models[j], arena, round, similarity=similarity |
| 206 | ) |
| 207 | except Exception as e: |
| 208 | print( |
| 209 | f"Error computing consistency for {models[i]} vs {models[j]} in {arena} at round {round}: {e}" |
| 210 | ) |
| 211 | continue |
| 212 | f.write( |
| 213 | json.dumps( |
| 214 | { |
| 215 | "model_a": models[i], |
no test coverage detected