* Check if a path is allowed according to robots.txt rules * @param rules - The parsed robots.txt rules * @param path - The path to check * @returns boolean indicating if access is allowed
(rules: RobotsRule[], path: string)
| 73 | * @returns boolean indicating if access is allowed |
| 74 | */ |
| 75 | function isPathAllowed(rules: RobotsRule[], path: string): boolean { |
| 76 | // Path should start with a slash |
| 77 | if (!path.startsWith("/")) { |
| 78 | path = "/" + path; |
| 79 | } |
| 80 | |
| 81 | // First find the applicable rule set (for * or for our user agent) |
| 82 | // We'll use * since we don't specify a specific user agent |
| 83 | let applicableRules = rules.find((rule) => rule.userAgent === "*"); |
| 84 | |
| 85 | // If no wildcard rules, check if any rules apply at all |
| 86 | if (!applicableRules && rules.length > 0) { |
| 87 | applicableRules = rules[0]; // Use the first rule as default |
| 88 | } |
| 89 | |
| 90 | // If no applicable rules or empty rules, allow access |
| 91 | if ( |
| 92 | !applicableRules || |
| 93 | (applicableRules.disallow.length === 0 && |
| 94 | applicableRules.allow.length === 0) |
| 95 | ) { |
| 96 | return true; |
| 97 | } |
| 98 | |
| 99 | // Check specific allow rules (these take precedence over disallow) |
| 100 | for (const allowPath of applicableRules.allow) { |
| 101 | if (path.startsWith(allowPath)) { |
| 102 | return true; |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // Check disallow rules |
| 107 | for (const disallowPath of applicableRules.disallow) { |
| 108 | if (disallowPath === "/" || path.startsWith(disallowPath)) { |
| 109 | return false; |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | // Default to allow if no disallow rules match |
| 114 | return true; |
| 115 | } |
| 116 | |
| 117 | /** |
| 118 | * Check if a specific URL is allowed according to robots.txt rules |