Method tokenize

lib/util/util/chat_input.py:47–48

(self, subject: "Subject")

class ModelInput(BaseModel):
    @abstractmethod
    def tokenize(self, subject: "Subject") -> list[int]:
        pass

    def token_strs(self, subject: "Subject") -> list[str]:
        return [subject.decode(x) for x in self.tokenize(subject)]