Function main

python3_cron_scripts/download_from_remote_database.py:288–396

Begin Main...

(logger=None)

def main(logger=None):
    """
    Begin Main...
    """
    if logger is None:
        logger = LoggingUtil.create_log(__name__)

    now = datetime.now()
    print("Starting: " + str(now))
    logger.info("Starting...")

    # Connect to the remote databases
    mongo_connector = MongoConnector.MongoConnector()
    rm_connector = RemoteMongoConnector.RemoteMongoConnector()
    dns_manager = DNSManager.DNSManager(mongo_connector)
    zones = ZoneManager.get_distinct_zones(mongo_connector)

    jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download")
    jobs_manager.record_job_start()

    remote_jobs_collection = rm_connector.get_jobs_connection()

    # Check the status of the Censys job on the remote database
    try:
        status = remote_jobs_collection.find_one({"job_name": "censys"})
    except:
        logger.error("Can not connect to remote database")
        jobs_manager.record_job_error()
        exit(1)

    if (
        status is not None
        and "status" in status
        and status["status"] != jobs_manager.COMPLETE
    ):
        logger.info("Censys scans status is not COMPLETE")
    elif (
        status is not None
        and "status" in status
        and status["status"] == jobs_manager.COMPLETE
    ):
        # Get connections to the relevant collections.
        censys_collection = mongo_connector.get_zgrab_443_data_connection()
        remote_censys_collection = rm_connector.get_zgrab_443_data_connection()

        download_censys_scan_info(logger, censys_collection, remote_censys_collection)

        # Tell the remote database that is safe to start processing the next Censys file
        remote_jobs_collection.update_one(
            {"job_name": "censys"},
            {"$currentDate": {"updated": True}, "$set": {"status": jobs_manager.READY}},
        )

    # Get connections to the relevant HTTPS collections.
    zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection()
    remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection()

    download_zgrab_info(