Function count_tokens

utils/tokenizer.py:129–134

Count tokens in text.

(text: str, tokenizer: Optional[TokenizerProtocol] = None)

def count_tokens(text: str, tokenizer: Optional[TokenizerProtocol] = None) -> int:
    """Count tokens in text."""
    if tokenizer is None:
        tokenizer = get_tokenizer()

    return len(tokenizer.encode(text))


# ============================================================================