Method _tokenize

linear_moe/tokenizer/tokenization_yi.py:106–108

Returns a tokenized string.

(self, text)

        return vocab

    def _tokenize(self, text):
        """Returns a tokenized string."""
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""