r"""Get model encoding from tiktoken. Args: value_for_tiktoken: Model value for tiktoken. Returns: tiktoken.Encoding: Model encoding.
(value_for_tiktoken: str)
| 49 | |
| 50 | |
| 51 | def get_model_encoding(value_for_tiktoken: str): |
| 52 | r"""Get model encoding from tiktoken. |
| 53 | |
| 54 | Args: |
| 55 | value_for_tiktoken: Model value for tiktoken. |
| 56 | |
| 57 | Returns: |
| 58 | tiktoken.Encoding: Model encoding. |
| 59 | """ |
| 60 | import tiktoken |
| 61 | |
| 62 | try: |
| 63 | encoding = tiktoken.encoding_for_model(value_for_tiktoken) |
| 64 | except KeyError: |
| 65 | if value_for_tiktoken in [ |
| 66 | ModelType.O1.value, |
| 67 | ModelType.O1_MINI.value, |
| 68 | ModelType.O1_PREVIEW.value, |
| 69 | ModelType.O3.value, |
| 70 | ModelType.O3_MINI.value, |
| 71 | ModelType.GPT_5.value, |
| 72 | ModelType.GPT_5_MINI.value, |
| 73 | ModelType.GPT_5_NANO.value, |
| 74 | ]: |
| 75 | encoding = tiktoken.get_encoding("o200k_base") |
| 76 | else: |
| 77 | logger.info("Model not found. Using cl100k_base encoding.") |
| 78 | encoding = tiktoken.get_encoding("cl100k_base") |
| 79 | return encoding |
| 80 | |
| 81 | |
| 82 | class BaseTokenCounter(ABC): |
no outgoing calls
no test coverage detected