(args)
| 106 | return data, best |
| 107 | |
| 108 | def main(args): |
| 109 | |
| 110 | #must pass in dataset arg for proper results |
| 111 | |
| 112 | os.makedirs(args.out_dir, exist_ok=True) |
| 113 | # setup plots |
| 114 | sns.set_style('darkgrid') |
| 115 | sns.set() |
| 116 | |
| 117 | frames = [] #array that collects dataframes from each file |
| 118 | if(args.dataset == "all"): |
| 119 | dataset_type = "*" |
| 120 | else: |
| 121 | dataset_type = args.dataset |
| 122 | |
| 123 | #gets all files that start with "resisc_" (still need a file with the baseline results) |
| 124 | # result_files = glob.glob(os.path.join(args.results_dir, dataset_type + "*.json"), recursive=True) |
| 125 | result_files1 = glob.glob(os.path.join(args.results_dir, dataset_type + "*crop*.json"), recursive=True) |
| 126 | result_files2 = glob.glob(os.path.join(args.results_dir, dataset_type + "*color*.json"), recursive=True) |
| 127 | result_files3 = glob.glob(os.path.join(args.results_dir, dataset_type + "*gray*.json"), recursive=True) |
| 128 | result_files = result_files1 + result_files2 + result_files3 |
| 129 | result_files.append(os.path.join(args.results_dir, dataset_type + "_results.json")) |
| 130 | |
| 131 | moco_baseline = imagenet_baseline = nobt_baseline = 0 |
| 132 | |
| 133 | for resfile in result_files: |
| 134 | with open(resfile, 'r') as infile: |
| 135 | raw_data = json.load(infile) |
| 136 | |
| 137 | print(resfile) |
| 138 | |
| 139 | types = [] #used for concatenating the resultzs from each basetrained model |
| 140 | data = pd.DataFrame(raw_data.values()) |
| 141 | |
| 142 | |
| 143 | #finds the relevant values for moco bt, imagenet supervised bt, and no bt |
| 144 | #mainly done to work around the baseline json file (has extra info that we don't want) |
| 145 | if (dataset_type + "_results.json") in resfile: |
| 146 | linear_data = data[data.result_type=='linear-eval'] |
| 147 | linear_data = linear_data[linear_data.variant=="linear-eval-lr"] |
| 148 | |
| 149 | data_moco = linear_data[data.basetrain=="moco_v2_800ep"] |
| 150 | data_moco = data_moco[data_moco.pretrain_iters=="5000"] |
| 151 | data_moco = data_moco[data_moco.pretrain_data==dataset_type] |
| 152 | data_moco, moco_baseline = reduce(data_moco) |
| 153 | |
| 154 | if args.basetrain == 'supervised': |
| 155 | data_imagenet = linear_data[data.basetrain=="imagenet_r50_supervised"] |
| 156 | data_imagenet = data_imagenet[data_imagenet.pretrain_iters=="50000"] |
| 157 | data_imagenet = data_imagenet[data_imagenet.pretrain_data==dataset_type] |
| 158 | data_imagenet, imagenet_baseline= reduce(data_imagenet) |
| 159 | types.append(data_imagenet) |
| 160 | |
| 161 | data_nobt = linear_data[data.basetrain=="no"] |
| 162 | data_nobt= data_nobt[data_nobt.pretrain_iters=="100000"] |
| 163 | data_nobt= data_nobt[data_nobt.pretrain_data==dataset_type] |
| 164 | data_nobt, nobt_baseline = reduce(data_nobt) |
| 165 |
no test coverage detected