Function _process_one_batch

explainshell/extraction/runner.py:288–466

Process a single provider batch: submit -> poll -> collect -> finalize. Never raises for normal batch/file failures — returns FAILED entries instead. Only truly unexpected errors (``KeyboardInterrupt``, ``SystemExit``) propagate.

(
    extractor: BatchExtractor,
    manifest: BatchManifestWriter,
    batch_idx: int,
    total_batches: int,
    batch_items: list[WorkItem],
    inflight: _InflightBatches,
)

def _process_one_batch(
    extractor: BatchExtractor,
    manifest: BatchManifestWriter,
    batch_idx: int,
    total_batches: int,
    batch_items: list[WorkItem],
    inflight: _InflightBatches,
) -> _BatchOutput:
    """Process a single provider batch: submit -> poll -> collect -> finalize.

    Never raises for normal batch/file failures — returns FAILED entries
    instead.  Only truly unexpected errors (``KeyboardInterrupt``,
    ``SystemExit``) propagate.
    """
    bp = extractor.batch_provider
    entries: list[ExtractionResult] = []
    finalized: set[str] = set()
    usage = TokenUsage()
    job_id: str | None = None
    batch_error: str | None = None

    def _add(entry: ExtractionResult) -> None:
        finalized.add(entry.gz_path)
        entries.append(entry)

    try:
        # Build batch requests.
        requests: list[BatchEntry] = []
        for item_idx, (gz_path, prepared) in enumerate(batch_items):
            for chunk_idx, user_content in enumerate(prepared.requests):
                requests.append(BatchEntry(f"{item_idx}:{chunk_idx}", user_content))

        total_chars = sum(len(req.user_content) for req in requests)
        logger.info(
            "submitting batch %d/%d (%d requests, %s chars)...",
            batch_idx,
            total_batches,
            len(requests),
            f"{total_chars:,}",
        )

        client = bp.make_poll_client()
        job_id = bp.submit_batch(requests)
        logger.info("batch %d/%d submitted: %s", batch_idx, total_batches, job_id)
        inflight.register(bp, client, job_id)

        # Persist the batch ID immediately so it survives crashes/interrupts.
        manifest.record_batch(
            batch_idx=batch_idx,
            batch_id=job_id,
            status="submitted",
            files=[item.gz_path for item in batch_items],
        )

        try:
            completed_job = bp.poll_batch(
                client, job_id, poll_interval=30, stop_event=inflight.stop_event
            )