Analyze thinking patterns in the response. Extract tokens between and tags and count thought transitions. Args: response (str): The model's response text Returns: Dict: Analysis metrics including thinking tokens and thought transitions
(response: str)
| 133 | return None |
| 134 | |
| 135 | def analyze_thinking(response: str) -> Dict: |
| 136 | """ |
| 137 | Analyze thinking patterns in the response. |
| 138 | Extract tokens between <think> and </think> tags and count thought transitions. |
| 139 | |
| 140 | Args: |
| 141 | response (str): The model's response text |
| 142 | |
| 143 | Returns: |
| 144 | Dict: Analysis metrics including thinking tokens and thought transitions |
| 145 | """ |
| 146 | # Default result with zero values |
| 147 | result = { |
| 148 | "has_think_tags": False, |
| 149 | "thinking_tokens": 0, |
| 150 | "thinking_tokens_text": "", |
| 151 | "total_tokens": len(response.split()), |
| 152 | "thought_transitions": 0, |
| 153 | "transition_counts": {phrase: 0 for phrase in THOUGHT_TRANSITIONS}, |
| 154 | "transition_positions": [] |
| 155 | } |
| 156 | |
| 157 | # Extract content between <think> and </think> tags |
| 158 | think_pattern = re.compile(r'<think>(.*?)</think>', re.DOTALL) |
| 159 | think_match = think_pattern.search(response) |
| 160 | |
| 161 | if think_match: |
| 162 | thinking_text = think_match.group(1) |
| 163 | result["has_think_tags"] = True |
| 164 | result["thinking_tokens"] = len(thinking_text.split()) |
| 165 | result["thinking_tokens_text"] = thinking_text |
| 166 | |
| 167 | # Count thought transitions |
| 168 | position = 0 |
| 169 | for phrase in THOUGHT_TRANSITIONS: |
| 170 | # Find all occurrences of each transition phrase |
| 171 | for match in re.finditer(r'\b' + re.escape(phrase) + r'\b', thinking_text): |
| 172 | result["transition_counts"][phrase] += 1 |
| 173 | # Record the approximate token position of the transition |
| 174 | token_position = len(thinking_text[:match.start()].split()) |
| 175 | result["transition_positions"].append((phrase, token_position)) |
| 176 | |
| 177 | # Sort transition positions by token position |
| 178 | result["transition_positions"].sort(key=lambda x: x[1]) |
| 179 | |
| 180 | # Calculate total transitions |
| 181 | result["thought_transitions"] = sum(result["transition_counts"].values()) |
| 182 | |
| 183 | return result |
| 184 | |
| 185 | def analyze_logits_probs(logprobs_data: List[Dict]) -> Dict: |
| 186 | """ |
no test coverage detected