Function checkRobotsTxt

apps/api/src/controllers/tools.controller.ts:136–177

(
  baseUrl: string,
  path: string,
)

}

async function checkRobotsTxt(
  baseUrl: string,
  path: string,
): Promise<'allowed' | 'blocked' | 'error'> {
  try {
    const robotsUrl = new URL('/robots.txt', baseUrl).toString();
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), 5000);

    const response = await fetch(robotsUrl, {
      signal: controller.signal,
      headers: {
        'User-Agent': 'OpenPanel-SiteChecker/1.0',
      },
    });

    clearTimeout(timeout);

    if (!response.ok) {
      return 'error';
    }

    const text = await response.text();
    const rules = text.split('\n').filter((line) => {
      const trimmed = line.trim();
      return trimmed && !trimmed.startsWith('#');
    });

    for (const rule of rules) {
      if (rule.toLowerCase().startsWith('disallow:')) {
        const pattern = rule.substring(9).trim();
        if (pattern && path.includes(pattern.replace('*', ''))) {
          return 'blocked';
        }
      }
    }

    return 'allowed';
  } catch {
    return 'error';
  }
}

async function checkSitemap(baseUrl: string): Promise<boolean> {
  try {