Method load

packages/utils/src/internals/robots.ts:65–101

(
        url: string,
        proxyUrl?: string,
        options?: { signal?: AbortSignal; timeoutMillis?: number },
    )

    }

    protected static async load(
        url: string,
        proxyUrl?: string,
        options?: { signal?: AbortSignal; timeoutMillis?: number },
    ): Promise<RobotsTxtFile> {
        if (!HTTPError) {
            HTTPError = (await import('got-scraping')).HTTPError;
        }

        try {
            const response = await gotScraping({
                url,
                proxyUrl,
                method: 'GET',
                responseType: 'text',
                signal: options?.signal,
                ...(options?.timeoutMillis ? { timeout: { request: options.timeoutMillis } } : {}),
            });

            return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
        } catch (e) {
            if (e instanceof HTTPError && e.response.statusCode === 404) {
                return new RobotsTxtFile(
                    {
                        isAllowed() {
                            return true;
                        },
                        getSitemaps() {
                            return [];
                        },
                    },
                    proxyUrl,
                );
            }
            throw e;
        }
    }

    /**
     * Check if a URL should be crawled by robots.