Method get_tokenizer

src/MaxText/maxengine.py:1489–1501

Return a protobuf of tokenizer info, callable from Py or C++.

(self)

    }

  def get_tokenizer(self) -> TokenizerParameters:
    """Return a protobuf of tokenizer info, callable from Py or C++."""
    try:
      tokenizer_type_val = TokenizerType.DESCRIPTOR.values_by_name[self.config.tokenizer_type].number
      return TokenizerParameters(
          path=self.config.tokenizer_path,
          tokenizer_type=tokenizer_type_val,
          access_token=self.config.hf_access_token,
          use_chat_template=self.config.use_chat_template,
          extra_ids=0,
      )
    except KeyError as _:
      raise KeyError(f"Unsupported tokenizer type: {self.config.tokenizer_type}") from None

  def build_tokenizer(self, metadata: TokenizerParameters) -> tokenizer_api.Tokenizer:
    """Return a tokenizer"""