| 155 | callback(result, self.mlflow, self.run_id) |
| 156 | |
| 157 | class EvalExp: |
| 158 | DEFAULT_OUT = "~/.fara_eval" |
| 159 | |
| 160 | def __init__(self, ws = None, user = None, seed = None, max_n_images = 5, save_task_csv = False): |
| 161 | self.ws = ws or dummy_workspace.Workspace() |
| 162 | self.experiment_name = 'osagent_eval' |
| 163 | self.user = user or getpass.getuser().split('@')[0] |
| 164 | self.seed = seed |
| 165 | self.max_n_images = max_n_images |
| 166 | self.save_task_csv = save_task_csv |
| 167 | |
| 168 | def _clean_mlflow_key(self, key: str) -> str: |
| 169 | """Clean MLflow keys by replacing unsupported characters, Names may only contain alphanumerics, underscores (_), dashes (-), periods (.), spaces ( ), colon(:) and slashes (/).""" |
| 170 | return ''.join(c if c.isalnum() or c in ['_', '-', '.', ' ', ':', '/'] else '_' for c in key)[:250] |
| 171 | |
| 172 | def run(self, model_ref, system, benchmark, out_url, subsample = 1.0, redo_eval = False, run_id = '0', split = None, processes = -1, callbacks = None, eval_only = False, max_error_task_retries = 0): |
| 173 | # out_az = AzFolder.from_uri(out_url) |
| 174 | out_context = Path(out_url).expanduser() |
| 175 | model_ref.log_2_mlflow() |
| 176 | # with out_az.mount(readonly = False) as out_context, \ |
| 177 | with AzVllm(model_ref.model_url_to_start, model_ref.model_port, model_ref.device_id, model_ref.max_n_images, model_ref.dtype, model_ref.enforce_eager, model_ref.use_external_endpoint) as vllm: |
| 178 | mlflow.log_param('benchmark', benchmark.name) |
| 179 | cmd = ' '.join(sys.argv) |
| 180 | try: |
| 181 | mlflow.log_param('cmd', cmd) |
| 182 | except mlflow.exceptions.MlflowException as e: |
| 183 | log_long_command_as_params(cmd) |
| 184 | mlflow.log_param('out', out_context) |
| 185 | mlflow.log_param('out_url', out_url) |
| 186 | mlflow.log_param('dtype', model_ref.dtype) |
| 187 | mlflow.log_param('enforce_eager', model_ref.enforce_eager) |
| 188 | |
| 189 | # Log fn_call_template mapping if the system has this attribute |
| 190 | if hasattr(system, 'fn_call_template'): |
| 191 | log_fn_call_template_as_tag(system.fn_call_template) |
| 192 | |
| 193 | benchmark.download_dataset() |
| 194 | benchmark.load_dataset() |
| 195 | |
| 196 | examples = benchmark.get_split_examples(split) |
| 197 | |
| 198 | if self.seed is not None: |
| 199 | random.seed(self.seed) |
| 200 | np.random.seed(self.seed) |
| 201 | |
| 202 | if 0.0 < subsample < 1.0: |
| 203 | examples = random.sample(examples, int(len(examples) * subsample)) |
| 204 | |
| 205 | mlflow.log_param("total examples", len(examples)) |
| 206 | original_run_id = run_id |
| 207 | run_id = f'runs/{system.hash()}/{model_ref.model_prefix}/{self.user}/{benchmark.exec_hash()}/{run_id or 0}' |
| 208 | |
| 209 | mlflow.log_param('run_id', run_id) |
| 210 | output_folder = out_context / run_id |
| 211 | (output_folder / benchmark.eval_hash()).mkdir(parents=True, exist_ok=True) |
| 212 | (output_folder / 'traj').mkdir(parents=True, exist_ok=True) |
| 213 | callback = Callback(callbacks = callbacks or []) |
| 214 | results = run_eval_multiple_examples_with_progress(examples, processes, output_folder, redo_eval, system, benchmark, callback, eval_only, max_error_task_retries) |