Function main

python3_cron_scripts/common_crawl_graph.py:218–331

Begin main...

(logger=None)

def main(logger=None):
    """
    Begin main...
    """
    if logger is None:
        logger = LoggingUtil.create_log(__name__)

    parser = argparse.ArgumentParser(
        description="Search the Common Crawl graph dataset for new domains"
    )
    parser.add_argument(
        "--url", metavar="URL", help="The URL for the latest vertices file"
    )
    parser.add_argument(
        "--save_location",
        metavar="LOCATION",
        default="./files",
        help="The directory for saving files",
    )
    args = parser.parse_args()

    CURRENT_FILE = CURRENT_FILE_LIST

    if args.url != None:
        CURRENT_FILE = args.url

    now = datetime.now()
    print("Starting: " + str(now))
    logger.info("Starting...")

    mongo_connector = MongoConnector.MongoConnector()
    dns_manager = DNSManager.DNSManager(mongo_connector)
    jobs_manager = JobsManager.JobsManager(mongo_connector, "common_crawl_graph")
    jobs_manager.record_job_start()

    reversed_zones = ZoneManager.get_reversed_zones(mongo_connector)

    alphabet = list(string.digits + string.ascii_lowercase)

    # Create a dictionary of the zones grouped by their first letter
    # This will allow us to reduce the number of comparisons in the alphabetized CC files.
    grouped_zones = {}
    for letter in alphabet:
        grouped_zones[letter] = []

    for zone in reversed_zones:
        first_letter = zone[0]
        grouped_zones[first_letter].append(zone)

    save_location = args.save_location
    if not save_location.endswith("/"):
        save_location = save_location + "/"

    compressed_download_list = download_file(logger, CURRENT_FILE, save_location)
    try:
        subprocess.check_call(["gunzip", "-f", compressed_download_list])
    except Exception as e:
        logger.error("Could not unzip download list")