Function taskQueue

js/botasaurus-js/src/task.ts:301–374

(options: TaskOptions<I>& { sequential?: boolean, skipDuplicateInput?: boolean })

    }
}
export function taskQueue<I=any>(options: TaskOptions<I>& { sequential?: boolean, skipDuplicateInput?: boolean }) {
    // Extract parallel from options - it controls queue-level concurrency, not passed to createTask
    const { parallel: parallelOption, ...taskOptions } = options
    const run = createTask<I>(taskOptions as TaskOptions<I>, true)
    const performTask = () => {
        let seenItems = new Set()
        let lastPromise: Promise<any> = Promise.resolve()
        const state = { promises: [] as any[], draining: false }
        let sequential = 'sequential' in options ? options.sequential : false
        const skipDuplicateInput = options.skipDuplicateInput ?? false
        
        // Create concurrency limiter for parallel mode
        const maxLimit = determineMaxLimit(parallelOption)

        let limit: pLimit.Limit
        if (maxLimit <=1) {
            sequential = true
        } else {
            limit = pLimit(maxLimit)
        }



        const cleanup = () => {
            state.promises = []
            state.draining = false
            seenItems.clear()
            lastPromise = Promise.resolve()
        }

        return {
            put: function (data: any, overrideOptions: Omit<TaskOptions<any>, 'run'> = {}) {
                const uniqueData = getUniqueItems(data, skipDuplicateInput, seenItems)
                
                // Handle case when all items are duplicates (skipDuplicateInput=true)
                if (uniqueData === __SKIP__) {
                    return Promise.resolve(null)
                }
                if (Array.isArray(uniqueData) && uniqueData.length === 0) {
                    return Promise.resolve([])
                }
                
                let promise: Promise<any> 
                if (sequential) {
                    // runs sequentially
                    promise = lastPromise.then(() => run(uniqueData, overrideOptions))
                    lastPromise = promise

                    state.promises.push(promise)
                    return promise.then(x=>x.result)
                } else {
                    if (Array.isArray(uniqueData)) {
                        promise = Promise.all(uniqueData.map(x=> limit(() => run(x, overrideOptions))))
                        state.promises.push(promise.then(results=>({originalData: uniqueData, result: results.map((x: any)=>x.result)})))
                        return promise.then(results=>results.map((x: any)=>x.result))
                    } else {
                        promise = limit(() => run(uniqueData, overrideOptions))