Crawl target website.
(self, target: Target)
| 24 | async def run(self, target: Target) -> ScanResult: |
| 25 | """Crawl target website.""" |
| 26 | result = self.create_result(target) |
| 27 | |
| 28 | base_url = self._build_url(target) |
| 29 | base_domain = urlparse(base_url).netloc |
| 30 | self.logger.info(f"Starting crawl of {base_url}") |
| 31 | |
| 32 | visited: set[str] = set() |
| 33 | to_visit: list[tuple[str, int]] = [(base_url, 0)] # (url, depth) |
| 34 | pages: list[dict] = [] |
| 35 | forms: list[dict] = [] |
| 36 | endpoints: set[str] = set() |
| 37 | external_links: set[str] = set() |
| 38 | |
| 39 | try: |
| 40 | async with HTTPClient() as client: |
| 41 | while to_visit and len(visited) < self.max_pages: |
| 42 | url, depth = to_visit.pop(0) |
| 43 | |
| 44 | if url in visited or depth > self.max_depth: |
| 45 | continue |
| 46 | |
| 47 | visited.add(url) |
| 48 | |
| 49 | try: |
| 50 | response = await client.get(url) |
| 51 | |
| 52 | if "text/html" not in response.headers.get("content-type", ""): |
| 53 | continue |
| 54 | |
| 55 | page_info = { |
| 56 | "url": str(response.url), |
| 57 | "status": response.status_code, |
| 58 | "title": None, |
| 59 | "depth": depth, |
| 60 | } |
| 61 | |
| 62 | soup = BeautifulSoup(response.text, "lxml") |
| 63 | |
| 64 | # Get title |
| 65 | title_tag = soup.find("title") |
| 66 | if title_tag: |
| 67 | page_info["title"] = title_tag.text.strip() |
| 68 | |
| 69 | pages.append(page_info) |
| 70 | |
| 71 | # Extract links |
| 72 | for link in soup.find_all("a", href=True): |
| 73 | href = link["href"] |
| 74 | full_url = urljoin(url, href) |
| 75 | parsed = urlparse(full_url) |
| 76 | |
| 77 | # Skip non-http, anchors, etc |
| 78 | if parsed.scheme not in ("http", "https"): |
| 79 | continue |
| 80 | |
| 81 | # Clean URL (remove fragment) |
| 82 | clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" |
| 83 | if parsed.query: |
no test coverage detected