Count the number of tokens in a text.
(text: str)
| 51 | enc = tiktoken.encoding_for_model("gpt-4") |
| 52 | |
| 53 | def count_tokens(text: str) -> int: |
| 54 | """ |
| 55 | Count the number of tokens in a text. |
| 56 | """ |
| 57 | length = len(enc.encode(text)) |
| 58 | # logger.debug(f"Number of tokens: {length}") |
| 59 | return length |
| 60 | |
| 61 | |
| 62 | # ------------------------------------------------------------ |
no outgoing calls
no test coverage detected