Wrap the process_instance_func to handle retries and errors. Retry an instance up to max_retries times if it fails (e.g., due to transient network/runtime issues).
(
process_instance_func: Callable[[pd.Series, EvalMetadata, bool], EvalOutput],
instance: pd.Series,
metadata: EvalMetadata,
use_mp: bool,
max_retries: int = 5,
)
| 270 | return _process_instance_wrapper(*args) |
| 271 | |
| 272 | def _process_instance_wrapper( |
| 273 | process_instance_func: Callable[[pd.Series, EvalMetadata, bool], EvalOutput], |
| 274 | instance: pd.Series, |
| 275 | metadata: EvalMetadata, |
| 276 | use_mp: bool, |
| 277 | max_retries: int = 5, |
| 278 | ) -> EvalOutput: |
| 279 | """Wrap the process_instance_func to handle retries and errors. |
| 280 | |
| 281 | Retry an instance up to max_retries times if it fails (e.g., due to transient network/runtime issues). |
| 282 | """ |
| 283 | if use_mp: |
| 284 | log_path = os.path.join(metadata.eval_output_dir, 'logs', f'agent_{metadata.model}_did_{instance["instance_id"]}.log') |
| 285 | logger = MetaChainLogger(log_path) |
| 286 | else: |
| 287 | logger = LoggerManager.get_logger() |
| 288 | for attempt in range(max_retries + 1): |
| 289 | try: |
| 290 | result = process_instance_func(instance, metadata, logger) |
| 291 | return result |
| 292 | except Exception as e: |
| 293 | error = str(e) |
| 294 | stacktrace = traceback.format_exc() |
| 295 | if attempt == max_retries: |
| 296 | logger.info(error, title='Error', color='red') |
| 297 | msg = ( |
| 298 | '-' * 10 |
| 299 | + '\n' |
| 300 | + f'Error in instance [{instance.instance_id}]: {error}. Stacktrace:\n{stacktrace}' |
| 301 | + '\n' |
| 302 | + f'[Encountered after {max_retries} retries. Please check the logs and report the issue.]' |
| 303 | + '-' * 10 |
| 304 | ) |
| 305 | # Raise an error after all retries & stop the evaluation |
| 306 | logger.info(error, title='Error', color='red') |
| 307 | raise RuntimeError( |
| 308 | f'Maximum error retries reached for instance {instance.instance_id}' |
| 309 | ) from e |
| 310 | msg = ( |
| 311 | '-' * 10 |
| 312 | + '\n' |
| 313 | + f'Error in instance [{instance.instance_id}]: {error}. Stacktrace:\n{stacktrace}' |
| 314 | + '\n' |
| 315 | + '-' * 10 |
| 316 | + f'[The above error occurred. Retrying... (attempt {attempt + 1} of {max_retries})]' |
| 317 | + '-' * 10 |
| 318 | + '\n' |
| 319 | ) |
| 320 | logger.info(msg, title='Error', color='red') |
| 321 | if use_mp: |
| 322 | print(msg) # use print to directly print to console |
| 323 | time.sleep(5) |
| 324 | |
| 325 | def update_progress( |
| 326 | result: EvalOutput, |
no test coverage detected