Begin Main...
(logger=None)
| 60 | |
| 61 | |
| 62 | def main(logger=None): |
| 63 | """ |
| 64 | Begin Main... |
| 65 | """ |
| 66 | if logger is None: |
| 67 | logger = LoggingUtil.create_log(__name__) |
| 68 | |
| 69 | now = datetime.now() |
| 70 | print("Starting: " + str(now)) |
| 71 | logger.info("Starting...") |
| 72 | |
| 73 | mongo_connector = MongoConnector.MongoConnector() |
| 74 | dns_manager = DNSManager.DNSManager(mongo_connector) |
| 75 | jobs_manager = JobsManager.JobsManager(mongo_connector, "extract_vt_domains") |
| 76 | |
| 77 | google_dns = GoogleDNS.GoogleDNS() |
| 78 | jobs_manager.record_job_start() |
| 79 | |
| 80 | round_two = [] |
| 81 | |
| 82 | zones = ZoneManager.get_distinct_zones(mongo_connector) |
| 83 | |
| 84 | vt_collection = mongo_connector.get_virustotal_connection() |
| 85 | vt_results = vt_collection.find( |
| 86 | {"subdomains": {"$exists": True}}, {"zone": 1, "subdomains": 1} |
| 87 | ).batch_size(20) |
| 88 | |
| 89 | input_list = [] |
| 90 | |
| 91 | # For each result found in the first pass across VirusTotal |
| 92 | for result in vt_results: |
| 93 | # Pause to prevent DoS-ing of Google's HTTPS DNS Service |
| 94 | time.sleep(1) |
| 95 | |
| 96 | for hostname in result["subdomains"]: |
| 97 | ips = google_dns.fetch_DNS_records(hostname) |
| 98 | |
| 99 | if ips is not None and ips != []: |
| 100 | for ip_addr in ips: |
| 101 | temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) |
| 102 | if temp_zone is not None: |
| 103 | record = {"fqdn": ip_addr["fqdn"]} |
| 104 | record["zone"] = temp_zone |
| 105 | record["created"] = datetime.now() |
| 106 | record["type"] = ip_addr["type"] |
| 107 | record["value"] = ip_addr["value"] |
| 108 | record["status"] = "unknown" |
| 109 | input_list.append(record) |
| 110 | |
| 111 | if ip_addr["type"] == "cname" and is_tracked_zone( |
| 112 | ip_addr["value"], zones |
| 113 | ): |
| 114 | add_to_list(ip_addr["value"], round_two) |
| 115 | else: |
| 116 | logger.warning("Failed IP Lookup for: " + hostname) |
| 117 | |
| 118 | dead_dns_collection = mongo_connector.get_dead_dns_connection() |
| 119 |
no test coverage detected