Method tokenize

examples/server/server.py:11934–11958

(
        self,
        text: str,
        *,
        add_bos: bool = True,
        special: bool = True,
    )

        )

    def tokenize(
        self,
        text: str,
        *,
        add_bos: bool = True,
        special: bool = True,
    ) -> List[int]:
        encoded = text.encode("utf-8")
        capacity = max(32, len(encoded) + 32)
        while True:
            tokens = (llama_cpp.llama_token * capacity)()
            count = int(
                llama_cpp.llama_tokenize(
                    self.vocab,
                    encoded,
                    len(encoded),
                    tokens,
                    capacity,
                    add_bos,
                    special,
                )
            )
            if count >= 0:
                return [int(tokens[index]) for index in range(count)]
            capacity = max(capacity * 2, -count)

    def build_prompt_tokens(self, prompt: str, suffix: Optional[str]) -> List[int]:
        if suffix is None: