Fallback token counter using `cl100k_base`.
| 271 | return total |
| 272 | |
| 273 | class DefaultTokenCounter(TokenCounter): |
| 274 | """Fallback token counter using `cl100k_base`.""" |
| 275 | |
| 276 | def __init__(self, model_name: str): |
| 277 | """Create a fallback token counter. |
| 278 | |
| 279 | Args: |
| 280 | model_name: Model identifier (kept for API symmetry). |
| 281 | """ |
| 282 | self.model_name = model_name |
| 283 | self.encoding = tiktoken.get_encoding("cl100k_base") |
| 284 | |
| 285 | def count_tokens(self, text: str) -> int: |
| 286 | return len(self.encoding.encode(text)) |
| 287 | |
| 288 | def count_message_tokens(self, messages: list[dict]) -> int: |
| 289 | total = 0 |
| 290 | for msg in messages: |
| 291 | for fragment in _message_text_fragments(msg): |
| 292 | total += self.count_tokens(fragment) |
| 293 | return total |
| 294 | |
| 295 | class TokenUsageTracker: |
| 296 | """Factory-backed token usage accumulator.""" |
no outgoing calls
no test coverage detected