Function encode_text

autotest/utils/toolkit.py:33–38

(model_path: str, text: str)

def encode_text(model_path: str, text: str) -> list[int]:
    tokenizer = _load_tokenizer_cached(model_path)

    encoded = tokenizer.encode(text)

    return encoded