Method count_tokens

methods/base.py:112–116

Count tokens in text.

(self, text: str)

        self._context_id = context_id

    def count_tokens(self, text: str) -> int:
        """Count tokens in text."""
        if not text:
            return 0
        return len(self._tokenizer.encode(text))

    @property
    def memory_size(self) -> int: