Method evaluate_skill

plugins/plugin-eval/src/plugin_eval/engine.py:71–127

Run evaluation layers on a skill directory and return a result.

(self, skill_dir: Path)

    # ------------------------------------------------------------------

    def evaluate_skill(self, skill_dir: Path) -> PluginEvalResult:
        """Run evaluation layers on a skill directory and return a result."""
        skill = parse_skill(skill_dir)
        layers: list[LayerResult] = []

        # Layer 1: Static analysis (always runs)
        static_result = self._static.analyze_skill(skill)
        layers.append(static_result)

        # Layer 2: Judge (standard+ depth)
        if self.config.depth in (Depth.STANDARD, Depth.DEEP, Depth.THOROUGH):
            from plugin_eval.layers.judge import JudgeAnalyzer, JudgeConfig

            judge_config = JudgeConfig(
                judges=self.config.judges,
                concurrency=self.config.concurrency,
            )
            judge = JudgeAnalyzer(judge_config)

            # Layer 3: Monte Carlo (deep+ depth) — run together with judge when both active
            if self.config.depth in (Depth.DEEP, Depth.THOROUGH):
                from plugin_eval.layers.monte_carlo import MonteCarloAnalyzer, MonteCarloConfig

                n_runs = self.config.monte_carlo_n or (
                    100 if self.config.depth == Depth.THOROUGH else 50
                )
                mc_config = MonteCarloConfig(
                    n_runs=n_runs,
                    concurrency=self.config.concurrency,
                )
                mc = MonteCarloAnalyzer(mc_config)

                async def _run_llm_layers(
                    judge: JudgeAnalyzer,
                    mc: MonteCarloAnalyzer,
                    skill: ParsedSkill,
                ) -> tuple[LayerResult, LayerResult]:
                    judge_result = await judge.analyze_skill(skill)
                    mc_result = await mc.analyze_skill(skill)
                    return judge_result, mc_result

                judge_result, mc_result = asyncio.run(_run_llm_layers(judge, mc, skill))
                layers.append(judge_result)
                layers.append(mc_result)
            else:
                judge_result = asyncio.run(judge.analyze_skill(skill))
                layers.append(judge_result)

        composite = self._build_composite(layers)

        return PluginEvalResult(
            plugin_path=str(skill_dir),
            timestamp=datetime.now(UTC).isoformat(),
            config=self.config,
            layers=layers,
            composite=composite,
        )