(resfile)
| 14 | |
| 15 | |
| 16 | def analyze_resfile(resfile): |
| 17 | before_summary = {} |
| 18 | after_summary = {} |
| 19 | |
| 20 | res = json.load(open(resfile))["eval"] |
| 21 | total = [] |
| 22 | before_pass = [] |
| 23 | after_pass = [] |
| 24 | for v in res.values(): |
| 25 | total.append(v["nfiles"]) |
| 26 | bc = sum([r[0] == SUCCESS for r in v["base"]]) |
| 27 | before_pass.append(bc) |
| 28 | if v["plus"]: |
| 29 | after_pass.append( |
| 30 | sum( |
| 31 | [ |
| 32 | v["plus"][i][0] == v["base"][i][0] == SUCCESS |
| 33 | for i in range(len(v["plus"])) |
| 34 | ] |
| 35 | ) |
| 36 | ) |
| 37 | |
| 38 | total = np.array(total) |
| 39 | before_pass = np.array(before_pass) |
| 40 | after_pass = np.array(after_pass) |
| 41 | for k in [1, 10, 100]: |
| 42 | if total.min() >= k: |
| 43 | pass_at_k = estimate_pass_at_k(total, before_pass, k).mean() |
| 44 | before_summary[f"pass@{k}"] = pass_at_k * 100 # scale to % |
| 45 | for k in [1, 10, 100]: |
| 46 | if total.min() >= k: |
| 47 | pass_at_k = estimate_pass_at_k(total, after_pass, k).mean() |
| 48 | after_summary[f"pass@{k}"] = pass_at_k * 100 |
| 49 | |
| 50 | return before_summary, after_summary |
| 51 | |
| 52 | |
| 53 | def align_ampersands(str1, str2): |
no test coverage detected