Function main

python3_cron_scripts/whois_lookups.py:162–225

Begin Main...

(logger=None)

def main(logger=None):
    """
    Begin Main...
    """
    if logger is None:
        logger = LoggingUtil.create_log(__name__)

    now = datetime.now()
    print("Starting: " + str(now))
    logger.info("Starting...")

    mongo_connector = RemoteMongoConnector.RemoteMongoConnector()
    jobs_manager = JobsManager.JobsManager(mongo_connector, "whois_lookups")
    jobs_manager.record_job_start()

    # Collect the tracked zones...
    zones = get_zones(mongo_connector)

    whois_collection = mongo_connector.get_whois_connection()

    for zone in zones:
        # Ensure the zone contains at least one dot. This is left over from an old bug.
        if zone.find(".") > 0:
            logger.debug(zone)
            zone_result = mongo_connector.perform_find_one(
                whois_collection, {"zone": zone}
            )

            # If we haven't done a lookup in the past, try to collect the data.
            # A limit exists on the number of whois lookups you can perform so limit to new domains.
            if zone_result is None:
                do_whois_lookup(logger, zone, whois_collection)

    # The cap on the number of old entries to be updated.
    MAX_OLD_ENTRIES = 400

    # Grab entries that haven't been updated in 3 months
    last_week = datetime.now() - timedelta(days=90, hours=1)
    zone_result = whois_collection.find({"updated": {"$lte": last_week}}).batch_size(10)

    i = 0
    for result in zone_result:
        do_whois_lookup(logger, result["zone"], whois_collection)
        i = i + 1

        # Chances are that a lot of the entries were inserted on the same day.
        # This helps break updating old entries across different runs.
        if i > MAX_OLD_ENTRIES:
            break

    # Delete old entries
    last_year = datetime.now() - timedelta(days=365, hours=1)
    zone_result = mongo_connector.perform_find(
        whois_collection, {"updated": {"$lte": last_year}}, batch_size=10
    )
    for result in zone_result:
        whois_collection.delete_one({"_id": ObjectId(result["_id"])})