| 84 | const rootUrl = new URL(address); |
| 85 | |
| 86 | async function checkUrl( |
| 87 | url: string, |
| 88 | referrer: string, |
| 89 | ): Promise<number> { |
| 90 | const cached = checkedUrls.get(url); |
| 91 | if (cached !== undefined) return cached; |
| 92 | |
| 93 | // Mark as in-flight to avoid duplicate checks |
| 94 | checkedUrls.set(url, 0); |
| 95 | |
| 96 | try { |
| 97 | const res = await fetch(url, { |
| 98 | method: "HEAD", |
| 99 | headers: { "User-Agent": "fresh-link-checker" }, |
| 100 | redirect: "follow", |
| 101 | }); |
| 102 | checkedUrls.set(url, res.status); |
| 103 | if (res.status >= 400) { |
| 104 | failedLinks.push({ url, status: res.status, referrer }); |
| 105 | } |
| 106 | return res.status; |
| 107 | } catch { |
| 108 | checkedUrls.set(url, 0); |
| 109 | failedLinks.push({ url, status: 0, referrer }); |
| 110 | return 0; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | async function crawlPage(pageUrl: URL, referrer: string) { |
| 115 | const pathname = pageUrl.pathname; |