| 173 | } |
| 174 | |
| 175 | export function _batch<T>(size: number, iterable: T[]): T[][] { |
| 176 | const batches: T[][] = [] |
| 177 | let currentBatch: T[] = [] |
| 178 | |
| 179 | iterable.forEach((item) => { |
| 180 | currentBatch.push(item) |
| 181 | |
| 182 | if (currentBatch.length >= size) { |
| 183 | batches.push(currentBatch) |
| 184 | currentBatch = [] |
| 185 | } |
| 186 | }) |
| 187 | |
| 188 | if (currentBatch.length > 0) { |
| 189 | batches.push(currentBatch) |
| 190 | } |
| 191 | |
| 192 | return batches |
| 193 | } |
| 194 | |
| 195 | export function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[] { |
| 196 | const seen = new Set<string>() |