| 59 | |
| 60 | @classmethod |
| 61 | def run_acc(self, run_records): |
| 62 | run_records = run_records.filter(lambda r: |
| 63 | len(r['args']['test_envs']) == 1) |
| 64 | if not len(run_records): |
| 65 | return None |
| 66 | test_env = run_records[0]['args']['test_envs'][0] |
| 67 | test_out_acc_key = 'env{}_out_acc'.format(test_env) |
| 68 | test_in_acc_key = 'env{}_in_acc'.format(test_env) |
| 69 | chosen_record = run_records.sorted(lambda r: r['step'])[-1] |
| 70 | return { |
| 71 | 'val_acc': chosen_record[test_out_acc_key], |
| 72 | 'test_acc': chosen_record[test_in_acc_key] |
| 73 | } |
| 74 | |
| 75 | class IIDAccuracySelectionMethod(SelectionMethod): |
| 76 | """Picks argmax(mean(env_out_acc for env in train_envs))""" |