MCPcopy Create free account
hub / github.com/algorithmicsuperintelligence/optillm / calc_logit_stats

Function calc_logit_stats

scripts/eval_aime_benchmark.py:653–716  ·  view source on GitHub ↗
(attempts)

Source from the content-addressed store, hash-verified

651
652 # Function to calculate logit statistics for a group of attempts
653 def calc_logit_stats(attempts):
654 if not attempts:
655 return {
656 "count": 0,
657 "entropy": None,
658 "transitions": None
659 }
660
661 # Collect all entropy stats
662 entropy_means = []
663 entropy_stds = []
664 entropy_quartiles = []
665 transition_entropies = defaultdict(lambda: {"before": [], "after": []})
666
667 for attempt in attempts:
668 if attempt['logit_analysis'].get('entropy_stats') and attempt['logit_analysis']['entropy_stats'].get('mean'):
669 entropy_means.append(attempt['logit_analysis']['entropy_stats']['mean'])
670 entropy_stds.append(attempt['logit_analysis']['entropy_stats']['std'])
671
672 if attempt['logit_analysis']['entropy_stats'].get('quartiles'):
673 entropy_quartiles.append(attempt['logit_analysis']['entropy_stats']['quartiles'])
674
675 # Collect transition entropy data
676 if attempt['logit_analysis'].get('transition_entropy'):
677 for phrase, stats in attempt['logit_analysis']['transition_entropy'].items():
678 if stats.get('before_mean') is not None:
679 transition_entropies[phrase]["before"].append(stats['before_mean'])
680 if stats.get('after_mean') is not None:
681 transition_entropies[phrase]["after"].append(stats['after_mean'])
682
683 # Calculate average entropy quartiles
684 avg_quartiles = []
685 if entropy_quartiles:
686 # Ensure all quartile lists have the same length
687 max_quartiles = max(len(q) for q in entropy_quartiles)
688 padded_quartiles = [q + [0] * (max_quartiles - len(q)) for q in entropy_quartiles]
689
690 # Calculate average for each quartile position
691 for i in range(max_quartiles):
692 quartile_values = [q[i] for q in padded_quartiles if i < len(q)]
693 avg_quartiles.append(statistics.mean(quartile_values) if quartile_values else 0)
694
695 # Calculate statistics for transitions
696 transition_stats = {}
697 for phrase, values in transition_entropies.items():
698 if values["before"] and values["after"]:
699 before_mean = statistics.mean(values["before"])
700 after_mean = statistics.mean(values["after"])
701 transition_stats[phrase] = {
702 "before_mean": before_mean,
703 "after_mean": after_mean,
704 "entropy_change": after_mean - before_mean,
705 "count": len(values["before"])
706 }
707
708 return {
709 "count": len(attempts),
710 "entropy": {

Callers 1

analyze_resultsFunction · 0.85

Calls

no outgoing calls

Tested by

no test coverage detected