()
| 28 | |
| 29 | |
| 30 | def main(): |
| 31 | args = parse_args() |
| 32 | if not args.no_fit: |
| 33 | fit_models(args) |
| 34 | |
| 35 | expdir = os.path.join("results/train", args.dataset, "exp_quantiative") |
| 36 | seeds = [s for s in os.listdir(expdir) if s.isdigit()] |
| 37 | |
| 38 | stats = [] |
| 39 | for seed in seeds: |
| 40 | runs = [r for r in os.listdir(os.path.join(expdir, seed)) if r.endswith(".json")] |
| 41 | |
| 42 | for run in runs: |
| 43 | runname, ext = os.path.splitext(run) |
| 44 | if args.dataset == "inat2018": |
| 45 | pe, nn = runname.split("-")[:2] # format "direct-mlp-val_loss=6.56_inat2018_result.jsom" |
| 46 | else: |
| 47 | pe, nn = runname.split("-") # format "direct-mlp.json" |
| 48 | |
| 49 | with open(os.path.join(expdir, seed, runname + ".json")) as f: |
| 50 | stat = json.load(f) |
| 51 | |
| 52 | if isinstance(stat, list): |
| 53 | stat = stat[0] |
| 54 | |
| 55 | if args.dataset == "inat2018": |
| 56 | # rename some metrics to match checkerboard and landoceandataset results |
| 57 | stat["accuracy"] = stat["val_acc"] |
| 58 | |
| 59 | stat.update(dict( |
| 60 | pe = pe, |
| 61 | nn = nn, |
| 62 | seed = int(seed) |
| 63 | )) |
| 64 | stats.append(stat) |
| 65 | |
| 66 | df = pd.DataFrame(stats) |
| 67 | csvfile = os.path.join(expdir, "runs.csv") |
| 68 | df.to_csv(csvfile) |
| 69 | |
| 70 | df_mean = df[["pe", "nn", "accuracy", "seed"]].groupby(["pe", "nn"]).mean()["accuracy"] |
| 71 | df_std = df[["pe", "nn", "accuracy", "seed"]].groupby(["pe", "nn"]).std()["accuracy"] |
| 72 | |
| 73 | # iterate over every entry |
| 74 | cells = [] |
| 75 | for mean, std in zip(df_mean, df_std): |
| 76 | cells.append(f"${mean*100:.1f} \pm {std*100:.1f}$") |
| 77 | |
| 78 | df_cells = pd.Series(cells) |
| 79 | df_cells.name = "accuracy" |
| 80 | df_cells.index = df_mean.index |
| 81 | |
| 82 | mean_table = pd.pivot_table(df_mean.reset_index(), "accuracy", "pe", "nn") |
| 83 | std_table = pd.pivot_table(df_std.reset_index(), "accuracy", "pe", "nn") |
| 84 | cells_table = pd.pivot(df_cells.reset_index(), values="accuracy", columns="nn", index="pe") |
| 85 | |
| 86 | csvfile = os.path.join(expdir, f"{args.dataset}_mean_table.csv") |
| 87 | mean_table.to_csv(csvfile) |
no test coverage detected