Function fetchFileWithRobotsTxtCheck

src/api/utils/robotsTxt.ts:173–204

(
  url: string,
  env: Env,
)

 * @returns File content or null if not allowed or not found
 */
export async function fetchFileWithRobotsTxtCheck(
  url: string,
  env: Env,
): Promise<{ content: string | null; blockedByRobots: boolean }> {
  try {
    const urlObj = new URL(url);
    // Create path from URL path + filename
    const path = urlObj.pathname;

    // Check robots.txt before attempting to fetch
    const isAllowed = await checkRobotsTxt(urlObj.hostname, path, env);

    if (!isAllowed) {
      console.log(`Access to ${url} disallowed by robots.txt`);
      return { content: null, blockedByRobots: true };
    }

    // If allowed, use cached content or fetch
    const content = await fetchUrlContent({
      url,
      format: "text",
    });

    return {
      content: content,
      blockedByRobots: false,
    };
  } catch (error) {
    console.warn(`Error fetching ${url}: ${error}`);
    return { content: null, blockedByRobots: false };
  }
}