Check if an S3 object exists at the given path. Parameters ---------- s3_file_path : str The S3 file path to check. Returns ------- bool True if the object exists, False otherwise. Raises ------ ClientError If there's an S3 error other t
(s3_file_path: str)
| 413 | |
| 414 | |
| 415 | def check_s3_object_exists(s3_file_path: str) -> bool: |
| 416 | """Check if an S3 object exists at the given path. |
| 417 | |
| 418 | Parameters |
| 419 | ---------- |
| 420 | s3_file_path : str |
| 421 | The S3 file path to check. |
| 422 | |
| 423 | Returns |
| 424 | ------- |
| 425 | bool |
| 426 | True if the object exists, False otherwise. |
| 427 | |
| 428 | Raises |
| 429 | ------ |
| 430 | ClientError |
| 431 | If there's an S3 error other than 404 (not found). |
| 432 | |
| 433 | """ |
| 434 | if not is_s3_enabled(): |
| 435 | logger.info("S3 not enabled, skipping object existence check", extra={"s3_file_path": s3_file_path}) |
| 436 | return False |
| 437 | |
| 438 | logger.info("Checking S3 object existence", extra={"s3_file_path": s3_file_path}) |
| 439 | _s3_ingest_lookup_counter.inc() |
| 440 | try: |
| 441 | s3_client = create_s3_client() |
| 442 | bucket_name = get_s3_bucket_name() |
| 443 | |
| 444 | # Use head_object to check if the object exists without downloading it |
| 445 | s3_client.head_object(Bucket=bucket_name, Key=s3_file_path) |
| 446 | except ClientError as err: |
| 447 | # Object doesn't exist if we get a 404 error |
| 448 | error_code = err.response.get("Error", {}).get("Code") |
| 449 | if error_code == "404": |
| 450 | logger.info( |
| 451 | "S3 object not found", |
| 452 | extra={ |
| 453 | "s3_file_path": s3_file_path, |
| 454 | "bucket_name": get_s3_bucket_name(), |
| 455 | "error_code": error_code, |
| 456 | }, |
| 457 | ) |
| 458 | _s3_ingest_miss_counter.inc() |
| 459 | return False |
| 460 | # Re-raise other errors (permissions, etc.) |
| 461 | raise |
| 462 | except Exception as exc: |
| 463 | # For any other exception, assume object doesn't exist |
| 464 | logger.info( |
| 465 | "S3 object check failed with exception, assuming not found", |
| 466 | extra={ |
| 467 | "s3_file_path": s3_file_path, |
| 468 | "bucket_name": get_s3_bucket_name(), |
| 469 | "exception": str(exc), |
| 470 | }, |
| 471 | ) |
| 472 | _s3_ingest_miss_counter.inc() |
no test coverage detected