Method _tokenize

EasyLM/models/llama/llama_model.py:1285–1287

Returns a tokenized string.

(self, text)

        return vocab

    def _tokenize(self, text):
        """Returns a tokenized string."""
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""