(self, text, process_fn=None)
| 83 | return self.EncodeAsIds(inputs, process_fn=process_fn) |
| 84 | |
| 85 | def EncodeAsIds(self, text, process_fn=None): |
| 86 | processed_text = text |
| 87 | if process_fn is not None: |
| 88 | processed_text = process_fn(processed_text) |
| 89 | ids = self.txt_tokenizer.encode(processed_text) |
| 90 | return [x + self.img_tokenizer.num_tokens for x in ids] |
| 91 | |
| 92 | def DecodeIds(self, ids): |
| 93 | ret, img_buffer, txt_buffer, ret_imgs = [], [], [], [] |
no test coverage detected