Function main

python3_cron_scripts/remove_expired_entries.py:157–339

Begin Main...

(logger=None)

def main(logger=None):
    """
    Begin Main...
    """
    if logger is None:
        logger = LoggingUtil.create_log(__name__)

    now = datetime.now()
    print("Starting: " + str(now))
    logger.info("Starting...")

    mongo_connector = MongoConnector.MongoConnector()
    all_dns_collection = mongo_connector.get_all_dns_connection()
    dns_manager = DNSManager.DNSManager(mongo_connector)
    GDNS = GoogleDNS.GoogleDNS()
    ip_manager = IPManager.IPManager(mongo_connector)

    jobs_manager = JobsManager.JobsManager(mongo_connector, "remove_expired_entries")
    jobs_manager.record_job_start()

    # zones = ZoneManager.get_distinct_zones(mongo_connector)

    parser = argparse.ArgumentParser(description="Remove expired entries from Marinus")
    parser.add_argument(
        "--source",
        required=False,
        help="Only run the script for a specific data source",
    )
    args = parser.parse_args()

    results = mongo_connector.perform_distinct(all_dns_collection, "sources.source")

    sources_list = None
    if args.source is not None and args.source != "":
        if args.source not in results:
            logger.error("FATAL: Unrecognized source value provided")
            exit(1)

        sources_list = [args.source]
    else:
        # The sources for which to remove expired entries
        sources_list = results

    sources = []
    for source in sources_list:
        temp = {}
        temp["name"] = source
        if "common_crawl" in source:
            temp["diff"] = -4
        else:
            temp["diff"] = -2

        sources.append(temp)

    # Before completely removing old entries, make an attempt to see if they are still valid.
    # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
    # Rather than throw away valid information, this will archive it.
    for entry in sources: