Function main

AskAgent/python/scraping/expBackOffCrawl.py:335–363

()

        self.stats.print_failure_summary()

def main():
    if len(sys.argv) < 3:
        print("Usage: python crawlUrls.py <input_file> <target_directory> [max_retries]")
        sys.exit(1)

    input_file = sys.argv[1]
    target_dir = sys.argv[2]

    # Parse additional arguments
    max_retries = 3  # Default

    # Check for max_retries
    if len(sys.argv) > 3:
        max_retries = int(sys.argv[3])

    # Read URLs
    with open(input_file) as f:
        urls = f.readlines()

    # Create and run crawler
    crawler = SimpleCrawler(
        target_dir=target_dir,
        max_retries=max_retries
    )

    print("Starting crawler with sequential processing")
    print(f"Max retries: {max_retries}")

    crawler.crawl_urls(urls)

if __name__ == "__main__":
    main()