Fallback token estimation method (when tiktoken is unavailable)
(self)
| 158 | return total_tokens |
| 159 | |
| 160 | def _estimate_tokens_fallback(self) -> int: |
| 161 | """Fallback token estimation method (when tiktoken is unavailable)""" |
| 162 | total_chars = 0 |
| 163 | for msg in self.messages: |
| 164 | if isinstance(msg.content, str): |
| 165 | total_chars += len(msg.content) |
| 166 | elif isinstance(msg.content, list): |
| 167 | for block in msg.content: |
| 168 | if isinstance(block, dict): |
| 169 | total_chars += len(str(block)) |
| 170 | |
| 171 | if msg.thinking: |
| 172 | total_chars += len(msg.thinking) |
| 173 | |
| 174 | if msg.tool_calls: |
| 175 | total_chars += len(str(msg.tool_calls)) |
| 176 | |
| 177 | # Rough estimation: average 2.5 characters = 1 token |
| 178 | return int(total_chars / 2.5) |
| 179 | |
| 180 | async def _summarize_messages(self): |
| 181 | """Message history summarization: summarize conversations between user messages when tokens exceed limit |