Method _encode

data_utils/tokenization.py:1210–1212

(self, text)

        self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()}

    def _encode(self, text):
        ids = self.text_tokenizer.encode(text)
        return ids

    def EncodeAsTokens(self, text, process_fn=None):
        processed_text = text