(self, text: str)
| 47 | log.info("Tokenizer loaded from %s (vocab=%d)", tokenizer_path, self._tok.get_vocab_size()) |
| 48 | |
| 49 | def encode(self, text: str) -> list[int]: |
| 50 | return self._tok.encode(text).ids |
| 51 | |
| 52 | def decode(self, ids: list[int]) -> str: |
| 53 | return self._tok.decode(ids) |
no outgoing calls
no test coverage detected