(self)
| 61 | |
| 62 | def decode(self, ids:list[int]) -> str: return b''.join(self._tok2bytes[tid] for tid in ids).decode(errors='replace') |
| 63 | def stream_decoder(self) -> typing.Callable[..., str]: |
| 64 | dec = codecs.getincrementaldecoder('utf-8')('replace') |
| 65 | def _decode(tid:int|None=None) -> str: return dec.decode(self._tok2bytes[tid]) if tid is not None else dec.decode(b'', final=True) |
| 66 | return _decode |
| 67 | def role(self, role:str): |
| 68 | if self.preset == 'olmo': return self.encode("<|" + role + "|>\n") # OLMoE Instruct format |
| 69 | if self.preset == 'kimi-k2': return self.encode("<|im_" + role + "|>" + role + "<|im_middle|>") |
no outgoing calls