Function encode_string_by_tiktoken

autoagent/memory/utils.py:4–9

(content: str, model_name: str = "gpt-4o")

ENCODER = None

def encode_string_by_tiktoken(content: str, model_name: str = "gpt-4o"):
    global ENCODER
    if ENCODER is None:
        ENCODER = tiktoken.encoding_for_model(model_name)
    tokens = ENCODER.encode(content)
    return tokens


def decode_tokens_by_tiktoken(tokens: list[int], model_name: str = "gpt-4o"):