| 378 | logger.info(f"Fail to log the uncommitted code of $CWD({os.getcwd()}) when run {cmd}.") |
| 379 | |
| 380 | def end_run(self, status: str = Recorder.STATUS_S): |
| 381 | assert status in [ |
| 382 | Recorder.STATUS_S, |
| 383 | Recorder.STATUS_R, |
| 384 | Recorder.STATUS_FI, |
| 385 | Recorder.STATUS_FA, |
| 386 | ], f"The status type {status} is not supported." |
| 387 | self.end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| 388 | if self.status != Recorder.STATUS_S: |
| 389 | self.status = status |
| 390 | if self.async_log is not None: |
| 391 | # Waiting Queue should go before mlflow.end_run. Otherwise mlflow will raise error |
| 392 | with TimeInspector.logt("waiting `async_log`"): |
| 393 | self.async_log.wait() |
| 394 | self.async_log = None |
| 395 | mlflow.end_run(status) |
| 396 | |
| 397 | def save_objects(self, local_path=None, artifact_path=None, **kwargs): |
| 398 | assert self.uri is not None, "Please start the experiment and recorder first before using recorder directly." |