Function download_amass_data

python3_cron_scripts/download_from_remote_database.py:175–204

Download the latest OWASP Amass information.

(
    logger, amass_collection, remote_amass_collection, dns_manager, zones
)

def download_amass_data(
    logger, amass_collection, remote_amass_collection, dns_manager, zones
):
    """
    Download the latest OWASP Amass information.
    """
    logger.info("Beginning Amass download")
    now = datetime.now()
    mirror_date = datetime.now() - timedelta(days=7, hours=9)
    amass_results = remote_amass_collection.find(
        {"timestamp": {"$gt": mirror_date}}, {"_id": 0}
    ).batch_size(50)

    google_dns = GoogleDNS.GoogleDNS()

    for result in amass_results:
        zone = check_zones(result["name"], zones)
        if zone is not None:
            time.sleep(1)
            if record_finding(logger, dns_manager, google_dns, zone, result):
                amass_collection.replace_one(
                    {"name": result["name"]}, result, upsert=True
                )

    # Establish a date four months back
    scrub_date = datetime.now() - timedelta(days=120, hours=9)

    # Remove data from scrub_date
    amass_collection.delete_many({"timestamp": {"$lte": scrub_date}})
    remote_amass_collection.delete_many({"timestamp": {"$lte": now}})


def download_jobs_status(logger, jobs_collection, remote_jobs_collection):