Method _load_tokenizer

opencompass/models/mixtral.py:62–64

(self, tokenizer_path: str)

        self.model = self.generator.model

    def _load_tokenizer(self, tokenizer_path: str):
        from mixtralkit.layers import Tokenizer
        self.tokenizer = Tokenizer(tokenizer_path)

    def generate(self, inputs: List[str], max_out_len: int) -> List[str]:
        prompt_tokens = []