Decode tokens to bytes. Args: tokens: List of token IDs Returns: Decoded bytes
(self, tokens: Sequence[int])
| 260 | # ==================== |
| 261 | |
| 262 | def decode_bytes(self, tokens: Sequence[int]) -> bytes: |
| 263 | """Decode tokens to bytes. |
| 264 | |
| 265 | Args: |
| 266 | tokens: List of token IDs |
| 267 | |
| 268 | Returns: |
| 269 | Decoded bytes |
| 270 | """ |
| 271 | try: |
| 272 | return bytes(self._core_bpe.decode_bytes(list(tokens))) |
| 273 | except Exception as e: |
| 274 | raise TokenDaggerError(f"Decoding failed: {e}") |
| 275 | |
| 276 | def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: |
| 277 | """Decode tokens to string. |