Function scrapingFunction

js/botasaurus-server-js/src/api-config.ts:122–351

(request: any, reply: any)

            : scraper.scraper_type;

        const scrapingFunction = async (request: any, reply: any) => {
            // Track if request was aborted by client
            let aborted = false;
            const onAbort = () => { aborted = true; };
            request.raw.on('close', onAbort);
            const isAborted = () => aborted;

            try {
                const params: Record<string, any> = {};
                for (const [key, value] of Object.entries(
                    request.query as any
                )) {
                    if (key.endsWith("[]")) {
                        params[key.slice(0, -2)] = Array.isArray(value)
                            ? value
                            : [value];
                    } else {
                        params[key] = value;
                    }
                }

                // Validate params against scraper's input definition
                const [validatedData, metadata, enableCache] = validateDirectCallRequest(
                    scraper.scraper_name,
                    params
                );

                // Check if scraper has split_task
                const splitTask = scraper.split_task;
                const scraperName = scraper.scraper_name;

                // If split_task exists, split the data
                let dataItems: any[];
                if (splitTask) {
                    dataItems = await splitTask(validatedData);
                } else {
                    dataItems = [validatedData];
                }

                const mt = isNotEmptyObject(metadata) ? { metadata } : {};
                let shouldDecrementCapacity = false;

                function restoreCapacity() {
                    if (shouldDecrementCapacity) {
                        getExecutor().decrementCapacity(key);
                        shouldDecrementCapacity = false;
                    }
                }

                // Collect results with metadata
                let collectedResults: Array<{
                    isFromCache: boolean;
                    isDontCache: boolean;
                    result: any;
                    cacheKey?: string;
                }> = [];
                try {
                    // Execute function for each data item