Function RunExperimentsAnalyze

pkg/cli/experiments_command.go:233–281

RunExperimentsAnalyze analyzes a specific experiment branch.

(config ExperimentsAnalyzeConfig)

// RunExperimentsAnalyze analyzes a specific experiment branch.
func RunExperimentsAnalyze(config ExperimentsAnalyzeConfig) error {
	experimentsLog.Printf("Analyzing experiment: name=%s, repo=%s, json=%v",
		config.ExperimentName, config.RepoOverride, config.JSONOutput)

	branchName := experimentsBranchPrefix + config.ExperimentName

	// Load experiment configs from the workflow frontmatter to enrich the statistical output
	// with hypothesis text, analysis_type, min_samples, and guardrail thresholds.
	// Config loading is best-effort: failures are silently ignored and analysis falls back to
	// defaults (min_samples=20, equal expected proportions, no hypothesis displayed).
	// This ensures the command remains functional even when the workflow .md file is absent
	// (e.g., when analysing experiments from a remote repository without the workflow checked out).
	var experimentConfigs map[string]*workflow.ExperimentConfig
	if config.RepoOverride != "" {
		experimentConfigs = loadRemoteExperimentConfigs(config.RepoOverride, config.ExperimentName)
	} else {
		experimentConfigs = loadLocalExperimentConfigs(config.ExperimentName)
	}
	experimentsLog.Printf("Loaded %d experiment config(s) for %s", len(experimentConfigs), config.ExperimentName)

	var details *ExperimentDetails
	var err error

	if config.RepoOverride != "" {
		details, err = fetchRemoteExperimentDetails(config.RepoOverride, branchName, config.ExperimentName)
	} else {
		details, err = fetchLocalExperimentDetails(branchName, config.ExperimentName)
	}

	if err != nil {
		fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error()))
		return nil
	}

	// Compute statistical analyses for each named experiment.
	details.Analyses = computeExperimentAnalyses(details.Experiments, experimentConfigs)

	if config.JSONOutput {
		jsonBytes, err := json.MarshalIndent(details, "", "  ")
		if err != nil {
			return fmt.Errorf("failed to marshal JSON: %w", err)
		}
		fmt.Fprintln(os.Stdout, string(jsonBytes))
		return nil
	}

	printExperimentDetails(details)
	return nil
}

// computeExperimentAnalyses computes statistical analyses for all named experiments.
// configs maps experiment names to their configuration; values may be nil.