Function requestHandler

test/e2e/request-queue-with-concurrency/actor/main.js:28–48

({ request, addRequests, pushData })

    const crawler = new BasicCrawler({
        requestQueue,
        async requestHandler({ request, addRequests, pushData }) {
            await setTimeout(Math.random() * 200); // Simulate some work

            const url = new URL(request.url);
            const index = Number(url.pathname.slice(1));
            const newUrls = [];

            for (const newIndex of [index + 1, index + 2, 2 * index]) {
                if (newIndex <= input.limit) {
                    url.pathname = `/${newIndex + 1}`;
                    newUrls.push(url.toString());
                }
            }

            log.info(`Enqueueing ${newUrls.length} new urls`);
            if (newUrls.length > 0) {
                await addRequests(newUrls);
            }

            await pushData({ url });
        },
    });

    await crawler.run(['https://example.tld/1']);