Analyze token probability distributions and entropy patterns. Args: logprobs_data: List of dictionaries containing token and logprob information Returns: Dict: Analysis metrics including entropy statistics
(logprobs_data: List[Dict])
| 183 | return result |
| 184 | |
| 185 | def analyze_logits_probs(logprobs_data: List[Dict]) -> Dict: |
| 186 | """ |
| 187 | Analyze token probability distributions and entropy patterns. |
| 188 | |
| 189 | Args: |
| 190 | logprobs_data: List of dictionaries containing token and logprob information |
| 191 | |
| 192 | Returns: |
| 193 | Dict: Analysis metrics including entropy statistics |
| 194 | """ |
| 195 | if not logprobs_data: |
| 196 | return { |
| 197 | "entropy_stats": None, |
| 198 | "transition_entropy": None, |
| 199 | "token_count": 0 |
| 200 | } |
| 201 | |
| 202 | token_entropies = [] |
| 203 | token_probs = [] |
| 204 | token_texts = [] |
| 205 | |
| 206 | # Process each token's logprobs |
| 207 | for token_info in logprobs_data: |
| 208 | if not token_info.get("top_logprobs"): |
| 209 | continue |
| 210 | |
| 211 | # Extract probabilities from logprobs |
| 212 | probs = [] |
| 213 | for token, logprob in token_info["top_logprobs"].items(): |
| 214 | probs.append(math.exp(logprob)) |
| 215 | |
| 216 | # Normalize probabilities to sum to 1 |
| 217 | total_prob = sum(probs) |
| 218 | if total_prob > 0: |
| 219 | probs = [p/total_prob for p in probs] |
| 220 | |
| 221 | # Calculate entropy: -sum(p_i * log(p_i)) |
| 222 | entropy = -sum(p * math.log2(p) if p > 0 else 0 for p in probs) |
| 223 | token_entropies.append(entropy) |
| 224 | token_probs.append(probs[0] if probs else 0) # Store top token probability |
| 225 | token_texts.append(token_info["token"]) |
| 226 | |
| 227 | # Analyze entropy changes around thought transitions |
| 228 | transition_entropy = {} |
| 229 | |
| 230 | for phrase in THOUGHT_TRANSITIONS: |
| 231 | # Find indices where this transition phrase begins |
| 232 | transition_indices = [] |
| 233 | |
| 234 | # Simple approach: find where token texts match the start of the phrase |
| 235 | for i, token in enumerate(token_texts): |
| 236 | if phrase.startswith(token) and i < len(token_texts) - 1: |
| 237 | # Check if this could be the start of the transition phrase |
| 238 | # This is a simplification; more complex matching would require full tokenization |
| 239 | transition_indices.append(i) |
| 240 | |
| 241 | # Analyze entropy changes around transitions |
| 242 | if transition_indices: |