| 342 | * @returns Array of results |
| 343 | */ |
| 344 | export async function processInBatches<T, R>( |
| 345 | items: T[], |
| 346 | batchSize: number, |
| 347 | processor: (item: T, index: number) => Promise<R>, |
| 348 | onBatchComplete?: (completed: number, total: number) => void |
| 349 | ): Promise<R[]> { |
| 350 | const results: R[] = []; |
| 351 | |
| 352 | for (let i = 0; i < items.length; i += batchSize) { |
| 353 | const batch = items.slice(i, Math.min(i + batchSize, items.length)); |
| 354 | const batchResults = await Promise.all( |
| 355 | batch.map((item, idx) => processor(item, i + idx)) |
| 356 | ); |
| 357 | results.push(...batchResults); |
| 358 | |
| 359 | if (onBatchComplete) { |
| 360 | onBatchComplete(Math.min(i + batchSize, items.length), items.length); |
| 361 | } |
| 362 | |
| 363 | // Allow GC between batches |
| 364 | if (global.gc) { |
| 365 | global.gc(); |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | return results; |
| 370 | } |
| 371 | |
| 372 | /** |
| 373 | * Simple mutex lock for preventing concurrent operations |