Method evaluate_plugin

plugins/plugin-eval/src/plugin_eval/engine.py:129–159

Run evaluation on an entire plugin directory (all skills + agents). Note: Plugin-level evaluation currently only runs Layer 1 (static). Judge and Monte Carlo require per-skill evaluation. The confidence label is always "Estimated" regardless of requested depth.

(self, plugin_dir: Path)

        )

    def evaluate_plugin(self, plugin_dir: Path) -> PluginEvalResult:
        """Run evaluation on an entire plugin directory (all skills + agents).

        Note: Plugin-level evaluation currently only runs Layer 1 (static).
        Judge and Monte Carlo require per-skill evaluation. The confidence
        label is always "Estimated" regardless of requested depth.
        """
        layers: list[LayerResult] = []

        # Layer 1: Static analysis of whole plugin
        static_result = self._static.analyze_plugin(plugin_dir)
        layers.append(static_result)

        # Plugin-level composite uses overall static score mapped to all
        # static-measurable dimensions (plugin result lacks per-dimension breakdown)
        static_overall = static_result.score
        dimension_scores = {dim: static_overall for dim in STATIC_TO_DIMENSION.values()}
        anti_pattern_count = len(static_result.anti_patterns)
        composite = self._assemble_composite(dimension_scores, anti_pattern_count)

        # Plugin-level eval only has static data — always "Estimated"
        # regardless of requested depth (judge/MC are per-skill only)
        composite.confidence_label = Depth.QUICK.confidence_label

        return PluginEvalResult(
            plugin_path=str(plugin_dir),
            timestamp=datetime.now(UTC).isoformat(),
            config=self.config,
            layers=layers,
            composite=composite,
        )

    # ------------------------------------------------------------------
    # Composite construction