Function robotsHandler

api/robots-txt.js:17–30

(url)

};

const robotsHandler = async (url) => {
  const { protocol, hostname } = parseTarget(url);
  const host = hostname.includes(':') ? `[${hostname}]` : hostname;
  try {
    const res = await httpGet(`${protocol}//${host}/robots.txt`);
    const parsed = parseRobotsTxt(res.data || '');
    return parsed.robots.length ? parsed : { skipped: 'No robots.txt rules found for this host' };
  } catch (error) {
    if (error.response?.status === 404) {
      return { skipped: 'No robots.txt file present on this host' };
    }
    return upstreamError(error, 'robots.txt fetch');
  }
};

export const handler = middleware(robotsHandler);
export default handler;