Function takeIndexableWithinCap

apps/sim/connectors/utils.ts:207–230

(
  items: T[],
  isSkipped: (item: T) => boolean,
  max: number,
  alreadyIndexed: number
)

 *   that count toward the cap), and whether the cap is now reached
 */
export function takeIndexableWithinCap<T>(
  items: T[],
  isSkipped: (item: T) => boolean,
  max: number,
  alreadyIndexed: number
): { documents: T[]; indexableCount: number; capReached: boolean } {
  if (max <= 0) {
    let indexableCount = 0
    for (const item of items) {
      if (!isSkipped(item)) indexableCount += 1
    }
    return { documents: items, indexableCount, capReached: false }
  }

  const remaining = max - alreadyIndexed
  const documents: T[] = []
  let indexableCount = 0
  for (const item of items) {
    if (indexableCount >= remaining) break
    documents.push(item)
    if (!isSkipped(item)) indexableCount += 1
  }
  return { documents, indexableCount, capReached: alreadyIndexed + indexableCount >= max }
}

/**
 * Raised by a connector when a file exceeds its size cap mid-download — i.e. the