Class WebCrawler

modules/webscanner/crawler.py:15–170

Crawl website to discover pages, forms, and endpoints.

class WebCrawler(WebScannerModule):
    """Crawl website to discover pages, forms, and endpoints."""

    name = "crawler"
    description = "Crawl website to discover pages, forms, and API endpoints"

    def __init__(self, max_pages: int = 100, max_depth: int = 3):
        super().__init__()
        self.max_pages = max_pages
        self.max_depth = max_depth

    async def run(self, target: Target) -> ScanResult:
        """Crawl target website."""
        result = self.create_result(target)

        base_url = self._build_url(target)
        base_domain = urlparse(base_url).netloc
        self.logger.info(f"Starting crawl of {base_url}")

        visited: set[str] = set()
        to_visit: list[tuple[str, int]] = [(base_url, 0)]  # (url, depth)
        pages: list[dict] = []
        forms: list[dict] = []
        endpoints: set[str] = set()
        external_links: set[str] = set()

        try:
            async with HTTPClient() as client:
                while to_visit and len(visited) < self.max_pages:
                    url, depth = to_visit.pop(0)

                    if url in visited or depth > self.max_depth:
                        continue

                    visited.add(url)

                    try:
                        response = await client.get(url)

                        if "text/html" not in response.headers.get("content-type", ""):
                            continue

                        page_info = {
                            "url": str(response.url),
                            "status": response.status_code,
                            "title": None,
                            "depth": depth,
                        }

                        soup = BeautifulSoup(response.text, "lxml")

                        # Get title
                        title_tag = soup.find("title")
                        if title_tag:
                            page_info["title"] = title_tag.text.strip()

                        pages.append(page_info)

                        # Extract links
                        for link in soup.find_all("a", href=True):