Function main

python3_cron_scripts/upload_akamai_data.py:57–130

(logger=None)

def main(logger=None):
    if logger is None:
        logger = LoggingUtil.create_log(__name__)

    now = datetime.now()
    print("Starting: " + str(now))
    logger.info("Starting...")

    mongo_connector = MongoConnector.MongoConnector()
    akamai_collection = mongo_connector.get_akamai_ips_connection()
    jobs_manager = JobsManager.JobsManager(mongo_connector, "upload_akamai_data")
    jobs_manager.record_job_start()

    # Clear previous data
    akamai_collection.delete_many({})

    AKAMAI_DATA = {}

    # Record the date that the data was updated.
    AKAMAI_DATA["created"] = datetime.now()

    try:
        response = _requests_retry_session().get(
            ZIP_FILE_LOCATION,
            stream=True,
            timeout=120,
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127"
            },
        )
    except requests.exceptions.HTTPError as e:
        logger.error("FATAL: HTTP error retrieving Akamai data: " + str(e))
        jobs_manager.record_job_error()
        exit(1)
    except Exception as ex:
        logger.error("FATAL: Error fetching Akamai data: " + str(ex))
        jobs_manager.record_job_error()
        exit(1)

    if response.status_code == 200:
        # Create the list of known IPV4 ranges.
        AKAMAI_DATA["ranges"] = []

        ipZip = ZipFile(BytesIO(response.content))
        for line in ipZip.open("akamai_ipv4_CIDRs.txt").readlines():
            AKAMAI_DATA["ranges"].append(
                {
                    "cidr": line.decode("utf-8").rstrip(),
                }
            )

        # Create the list of known IPV6 ranges.
        AKAMAI_DATA["ipv6_ranges"] = []
        for line in ipZip.open("akamai_ipv6_CIDRs.txt").readlines():
            AKAMAI_DATA["ipv6_ranges"].append(
                {
                    "cidr": line.decode("utf-8").rstrip(),
                }