Method crawl_recursive

core/scanner_engine.py:207–264

Recursively crawl the target website using parallel workers.

(self)

            return []
    
    def crawl_recursive(self) -> Set[str]:
        """Recursively crawl the target website using parallel workers."""
        self.state_manager.set_phase(PentestPhase.CRAWLING)
        console.print("[bold blue]🕷️  Starting web crawler...[/bold blue]")

        all_urls = set([self.target_url])
        queue = [(self.target_url, 0)]
        lock = threading.Lock()

        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            BarColumn(),
            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
            TimeElapsedColumn(),
            console=console
        ) as progress:

            task = progress.add_task(
                f"[cyan]Crawling (depth: {self.depth})...",
                total=None
            )

            with ThreadPoolExecutor(max_workers=min(self.threads, 10)) as executor:
                while queue:
                    # Grab a batch from the queue
                    batch = []
                    while queue and len(batch) < self.threads:
                        batch.append(queue.pop(0))

                    # Filter out items beyond depth
                    batch = [(url, depth) for url, depth in batch if depth < self.depth]
                    if not batch:
                        continue

                    # Crawl batch in parallel
                    futures = {executor.submit(self.crawl, url, depth): (url, depth) for url, depth in batch}

                    for future in futures:
                        try:
                            new_links = future.result(timeout=self.timeout * 3)
                            _, depth = futures[future]
                            with lock:
                                for link in new_links:
                                    if link not in all_urls:
                                        all_urls.add(link)
                                        queue.append((link, depth + 1))
                                        self.state_manager.update_urls(discovered=1)
                        except Exception as e:
                            logger.debug(f"Crawl worker error: {e}")

                    progress.update(
                        task,
                        description=f"[cyan]Crawling... Found {len(all_urls)} URLs"
                    )

        console.print(f"[green]✓[/green] Crawling complete. Found {len(all_urls)} URLs\n")
        return all_urls