Function parseRobotsTxt

src/api/utils/robotsTxt.ts:20–67

* Parse robots.txt content into structured rules * @param content - The content of robots.txt * @returns Array of parsed rules

(content: string)

 * @returns Array of parsed rules
 */
function parseRobotsTxt(content: string): RobotsRule[] {
  const lines = content.split("\n");
  const rules: RobotsRule[] = [];

  let currentRule: RobotsRule | null = null;

  for (const line of lines) {
    const trimmedLine = line.trim();

    // Skip comments and empty lines
    if (!trimmedLine || trimmedLine.startsWith("#")) {
      continue;
    }

    // Split into directive and value
    const [directive, ...valueParts] = trimmedLine.split(":");
    const value = valueParts.join(":").trim();

    if (!directive || !value) {
      continue;
    }

    const directiveLower = directive.trim().toLowerCase();

    // Start a new rule when encountering a User-agent directive
    if (directiveLower === "user-agent") {
      if (currentRule && currentRule.userAgent) {
        rules.push(currentRule);
      }
      currentRule = { userAgent: value, disallow: [], allow: [] };
    }
    // Add disallow paths
    else if (directiveLower === "disallow" && currentRule) {
      currentRule.disallow.push(value);
    }
    // Add allow paths
    else if (directiveLower === "allow" && currentRule) {
      currentRule.allow.push(value);
    }
  }

  // Add the last rule if exists
  if (currentRule && currentRule.userAgent) {
    rules.push(currentRule);
  }

  return rules;
}

/**
 * Check if a path is allowed according to robots.txt rules