()
| 92 | |
| 93 | |
| 94 | def main(): |
| 95 | parser = argparse.ArgumentParser(description="W&B Logging Script") |
| 96 | parser.add_argument("--entity", type=str, default="bert24", help="W&B entity name") |
| 97 | parser.add_argument("--meta-project", type=str, default="bert24-evals-meta", help="meta project name") |
| 98 | parser.add_argument("--model-name", type=str, default="bert24-large-v2", help="Model name") |
| 99 | parser.add_argument("--meta-run-id", type=str, help="ID of the meta run to update") |
| 100 | parser.add_argument("--meta-run-name", type=str, default="bert24-large-v2-evals", help="Meta run name") |
| 101 | |
| 102 | parser.add_argument("--source-project", type=str, default="bert24-large-v2-evals", help="project for eval runs") |
| 103 | parser.add_argument("--interval", type=int, default=60, help="Interval in minutes between data refresh") |
| 104 | parser.add_argument("--init-meta", action="store_true", help="Initialize a new meta run") |
| 105 | |
| 106 | args = parser.parse_args() |
| 107 | |
| 108 | # metadata information --- |
| 109 | args.task2metric_dict = { |
| 110 | "mnli": ["metrics/glue_mnli/MulticlassAccuracy", "metrics/glue_mnli_mismatched/MulticlassAccuracy"], |
| 111 | "ultrafeedback": ["metrics/long_context_ultrafeedback/UltrafeedbackAUROC"], |
| 112 | "mlmmlu_rookie_reserve": [ |
| 113 | "metrics/mlmmlu_rookie/MulticlassAccuracy", |
| 114 | "metrics/mlmmlu_reserve/MulticlassAccuracy", |
| 115 | ], |
| 116 | "wic": ["metrics/superglue_wic/MulticlassAccuracy"], |
| 117 | "boolq": ["metrics/superglue_boolq/MulticlassAccuracy"], |
| 118 | } |
| 119 | |
| 120 | args.metric2num_seeds = { |
| 121 | "metrics/glue_mnli/MulticlassAccuracy": 3, |
| 122 | "metrics/glue_mnli_mismatched/MulticlassAccuracy": 3, |
| 123 | "metrics/mlmmlu_rookie/MulticlassAccuracy": 3, |
| 124 | "metrics/mlmmlu_reserve/MulticlassAccuracy": 3, |
| 125 | "metrics/superglue_wic/MulticlassAccuracy": 3, |
| 126 | "metrics/superglue_boolq/MulticlassAccuracy": 3, |
| 127 | "metrics/long_context_ultrafeedback/UltrafeedbackAUROC": 2, |
| 128 | } |
| 129 | |
| 130 | args.all_metrics = [ |
| 131 | "metrics/glue_mnli/MulticlassAccuracy", |
| 132 | "metrics/glue_mnli_mismatched/MulticlassAccuracy", |
| 133 | # "metrics/mlmmlu_rookie/MulticlassAccuracy", |
| 134 | # "metrics/mlmmlu_reserve/MulticlassAccuracy", |
| 135 | "metrics/superglue_wic/MulticlassAccuracy", |
| 136 | "metrics/superglue_boolq/MulticlassAccuracy", |
| 137 | ] |
| 138 | |
| 139 | if args.init_meta: |
| 140 | meta_run_id = init_run(args) |
| 141 | print(f"Use this meta_run_id for future runs: {meta_run_id}") |
| 142 | return |
| 143 | |
| 144 | if not args.meta_run_id: |
| 145 | parser.error("--meta-run-id is required when not initializing a new meta run") |
| 146 | |
| 147 | schedule.every(args.interval).minutes.do(process_data, args) |
| 148 | process_data(args) # first run |
| 149 | |
| 150 | while True: |
| 151 | try: |
no test coverage detected