Function addRequests

packages/core/src/storages/request_provider.ts:275–384

* Adds requests to the queue in batches of 25. This method will wait till all the requests are added * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()` * if you don't want to block the processing, as those methods will only wait

(
        requestsLike: RequestsLike,
        options: RequestQueueOperationOptions = {},
    )

     * @param [options] Request queue operation options.
     */
    async addRequests(
        requestsLike: RequestsLike,
        options: RequestQueueOperationOptions = {},
    ): Promise<BatchAddRequestsResult> {
        checkStorageAccess();

        this.lastActivity = new Date();

        ow(
            requestsLike,
            ow.object
                .is((value: unknown) => isIterable(value) || isAsyncIterable(value))
                .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`),
        );
        ow(
            options,
            ow.object.exactShape({
                forefront: ow.optional.boolean,
                cache: ow.optional.boolean,
            }),
        );

        const { forefront = false, cache = true } = options;

        const uniqueKeyToCacheKey = new Map<string, string>();
        const getCachedRequestId = (uniqueKey: string) => {
            const cached = uniqueKeyToCacheKey.get(uniqueKey);

            if (cached) return cached;

            const newCacheKey = getRequestId(uniqueKey);
            uniqueKeyToCacheKey.set(uniqueKey, newCacheKey);

            return newCacheKey;
        };

        const results: BatchAddRequestsResult = {
            processedRequests: [],
            unprocessedRequests: [],
        };

        const requests: Request<Dictionary>[] = [];

        for await (const requestLike of requestsLike) {
            if (typeof requestLike === 'string') {
                requests.push(new Request({ url: requestLike }));
            } else if ('requestsFromUrl' in requestLike) {
                const fetchedRequests = await this._fetchRequestsFromUrl(requestLike as InternalSource);
                await this._addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);
            } else {
                requests.push(
                    requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions),
                );
            }
        }

        const requestsToAdd = new Map<string, Request>();