Method from

packages/utils/src/internals/robots.ts:61–63

* Allows providing the URL and robots.txt content explicitly instead of loading it from the target site. * @param url the URL for robots.txt file * @param content contents of robots.txt * @param [proxyUrl] a proxy to be used for fetching the robots.txt file

(url: string, content: string, proxyUrl?: string)

     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
     */
    static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile {
        return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
    }

    protected static async load(
        url: string,