Function run_config

scripts/holdout_param_sweep.py:199–251

(config: ExperimentConfig)

        }

    def run_config(config: ExperimentConfig) -> dict[str, Any]:
        original_low_threshold = ensemble_mod._LOW_ESCALATION_THRESHOLD
        ensemble_mod._LOW_ESCALATION_THRESHOLD = config.low_escalation_threshold
        ensemble_2sig = Ensemble(
            weights=[config.weights_a, config.weights_c],
            risk_tolerance=0.5,
            calibrator=calibrator,
        )
        ensemble_3sig = Ensemble(
            weights=list(THREE_SIGNAL_WEIGHTS),
            risk_tolerance=0.5,
            calibrator=calibrator,
        )

        def predict(row: dict[str, Any]) -> int:
            row_id = row["id"]
            vote_a = vote_a_by_id[row_id]
            vote_c = vote_c_by_threshold[config.classifier_fallback_threshold][row_id]

            if config.signal_b_conditional and len(row.get("messages", [])) >= 4:
                vote_b = vote_b_by_id[row_id]
                result = ensemble_3sig.decide([vote_a, vote_b, vote_c])
            else:
                result = ensemble_2sig.decide([vote_a, vote_c])

            return 1 if result.tier_id is None else result.tier_id

        try:
            predictor = FunctionPredictor(predict)
            per_row, errors, correct = evaluate_question_bank_rows(
                predictor,
                rows,
                predictor_label=_config_name(config),
            )
            summary = build_eval_summary(
                per_row=per_row,
                errors=errors,
                correct=correct,
                predictor_label=_config_name(config),
                shard=split_path,
                sample_mode=f"{split}_split",
                seed=42,
                proportional_quotas=None,
                benchmark_counts=benchmark_counts,
            )
            return _summarize_exact(
                config=config,
                per_row=per_row,
                errors=errors,
                summary=summary,
            )
        finally:
            ensemble_mod._LOW_ESCALATION_THRESHOLD = original_low_threshold

    baseline_config = ExperimentConfig()
    baseline = run_config(baseline_config)