Method run

packages/basic-crawler/src/internals/basic-crawler.ts:979–1098

* Runs the crawler. Returns a promise that resolves once all the requests are processed * and `autoscaledPool.isFinished` returns `true`. * * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for * running {@apilink BasicCrawler.addRequests|`c

(requests?: RequestsLike, options?: CrawlerRunOptions)

     * @param [options] Options for the request queue.
     */
    async run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics> {
        if (this.running) {
            throw new Error(
                'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
            );
        }

        const { purgeRequestQueue = true, ...addRequestsOptions } = options ?? {};

        if (this.hasFinishedBefore) {
            // When executing the run method for the second time explicitly,
            // we need to purge the default RQ to allow processing the same requests again - this is important so users can
            // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
            // ignored - as a failed requests is still handled.
            if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
                await this.requestQueue.drop();
                this.requestQueue = await this._getRequestQueue();
                this.requestManager = undefined;
                await this.initializeRequestManager();
            }

            this.stats.reset();
            await this.stats.resetStore();
            await this.sessionPool?.resetStore();
        }

        this.unexpectedStop = false;
        this.running = true;
        this.loggedPerRun.clear();

        await purgeDefaultStorages({
            onlyPurgeOnce: true,
            client: this.config.getStorageClient(),
            config: this.config,
        });

        if (requests) {
            await this.addRequests(requests, addRequestsOptions);
        }

        await this._init();
        await this.stats.startCapturing();
        const periodicLogger = this.getPeriodicLogger();
        // Don't await, we don't want to block the execution
        void this.setStatusMessage('Starting the crawler.', { level: 'INFO' });

        const sigintHandler = async () => {
            this.log.warning(
                'Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start',
            );
            await this._pauseOnMigration();
            await this.autoscaledPool!.abort();
        };

        // Attach a listener to handle migration and aborting events gracefully.
        const boundPauseOnMigration = this._pauseOnMigration.bind(this);
        process.once('SIGINT', sigintHandler);
        this.events.on(EventType.MIGRATING, boundPauseOnMigration);