Begin main...
(logger=None)
| 69 | |
| 70 | |
| 71 | def main(logger=None): |
| 72 | """ |
| 73 | Begin main... |
| 74 | """ |
| 75 | if logger is None: |
| 76 | logger = LoggingUtil.create_log(__name__) |
| 77 | |
| 78 | # Don't run if the search files script is working on the existing file. |
| 79 | if is_running("search_censys_files_new.py"): |
| 80 | now = datetime.now() |
| 81 | logger.warning("File search running: " + str(now)) |
| 82 | exit(0) |
| 83 | |
| 84 | # Record the start in the logs |
| 85 | now = datetime.now() |
| 86 | print("Starting: " + str(now)) |
| 87 | logger.info("Starting...") |
| 88 | |
| 89 | RMC = RemoteMongoConnector.RemoteMongoConnector() |
| 90 | jobs_collection = RMC.get_jobs_connection() |
| 91 | |
| 92 | # Obtain the timestamp of the last file that was downloaded. |
| 93 | last_timestamp = "0" |
| 94 | try: |
| 95 | f_time = open(TIMESTAMP_FILE, "r") |
| 96 | last_timestamp = f_time.readline() |
| 97 | f_time.close() |
| 98 | except FileNotFoundError: |
| 99 | last_timestamp = "0" |
| 100 | |
| 101 | # Get the meta data for the currently available file. |
| 102 | req = requests.get( |
| 103 | CENSYS_API + "data/ipv4", auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET) |
| 104 | ) |
| 105 | |
| 106 | if req.status_code != 200: |
| 107 | logger.warning( |
| 108 | "Error " + str(req.status_code) + ": Unable to query Censys Data API\n" |
| 109 | ) |
| 110 | logger.warning(req.text) |
| 111 | |
| 112 | time.sleep(60) |
| 113 | req = requests.get( |
| 114 | CENSYS_API + "data/ipv4", auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET) |
| 115 | ) |
| 116 | if req.status_code != 200: |
| 117 | logger.error("Error on IPv4 retry. Giving up...") |
| 118 | exit(1) |
| 119 | |
| 120 | data_json = json.loads(req.text) |
| 121 | |
| 122 | # Get the timestamp for the currently available file |
| 123 | timestamp = data_json["results"]["latest"]["timestamp"] |
| 124 | |
| 125 | # If it is the same file as last time, then don't download again. |
| 126 | if last_timestamp == timestamp: |
| 127 | logger.error("Already downloaded. Exiting...") |
| 128 | exit(0) |
no test coverage detected