r"""Constructor for the token counter for OpenAI models. Args: model (UnifiedModelType): Model type for which tokens will be counted.
(self, model: UnifiedModelType)
| 98 | |
| 99 | class OpenAITokenCounter(BaseTokenCounter): |
| 100 | def __init__(self, model: UnifiedModelType): |
| 101 | r"""Constructor for the token counter for OpenAI models. |
| 102 | |
| 103 | Args: |
| 104 | model (UnifiedModelType): Model type for which tokens will be |
| 105 | counted. |
| 106 | """ |
| 107 | self.model: str = model.value_for_tiktoken |
| 108 | |
| 109 | self.tokens_per_message: int |
| 110 | self.tokens_per_name: int |
| 111 | |
| 112 | if self.model == "gpt-3.5-turbo-0301": |
| 113 | # Every message follows <|start|>{role/name}\n{content}<|end|>\n |
| 114 | self.tokens_per_message = 4 |
| 115 | # If there's a name, the role is omitted |
| 116 | self.tokens_per_name = -1 |
| 117 | elif ("gpt-3.5-turbo" in self.model) or ("gpt-4" in self.model) or ("gpt-5" in self.model): |
| 118 | self.tokens_per_message = 3 |
| 119 | self.tokens_per_name = 1 |
| 120 | elif ("o1" in self.model) or ("o3" in self.model): |
| 121 | self.tokens_per_message = 2 |
| 122 | self.tokens_per_name = 1 |
| 123 | else: |
| 124 | # flake8: noqa :E501 |
| 125 | raise NotImplementedError( |
| 126 | "Token counting for OpenAI Models is not presently " |
| 127 | f"implemented for model {model}. " |
| 128 | "See https://github.com/openai/openai-python/blob/main/chatml.md " |
| 129 | "for information on how messages are converted to tokens. " |
| 130 | "See https://platform.openai.com/docs/models/gpt-4" |
| 131 | "or https://platform.openai.com/docs/models/gpt-3-5" |
| 132 | "for information about openai chat models." |
| 133 | ) |
| 134 | |
| 135 | self.encoding = get_model_encoding(self.model) |
| 136 | |
| 137 | def count_tokens_from_messages(self, messages: List[OpenAIMessage]) -> int: |
| 138 | r"""Count number of tokens in the provided message list with the |
nothing calls this directly
no test coverage detected