run preprocessing and encode inputs as Ids CANNOT contain command tokens
(self, inputs, process_fn=None)
| 74 | return self.num_tokens |
| 75 | |
| 76 | def __call__(self, inputs, process_fn=None): |
| 77 | """run preprocessing and encode inputs as Ids |
| 78 | CANNOT contain command tokens""" |
| 79 | if isinstance(inputs, torch.Tensor): # image |
| 80 | if len(inputs.shape) == 3: |
| 81 | inputs = inputs.unsqueeze(0) |
| 82 | return self.img_tokenizer.EncodeAsIds(inputs) |
| 83 | return self.EncodeAsIds(inputs, process_fn=process_fn) |
| 84 | |
| 85 | def EncodeAsIds(self, text, process_fn=None): |
| 86 | processed_text = text |
nothing calls this directly
no test coverage detected