Encode text using ordinary tokens only (no special tokens). Args: text: Text to encode Returns: List of token IDs
(self, text: str)
| 143 | # ==================== |
| 144 | |
| 145 | def encode_ordinary(self, text: str) -> list[int]: |
| 146 | """Encode text using ordinary tokens only (no special tokens). |
| 147 | |
| 148 | Args: |
| 149 | text: Text to encode |
| 150 | |
| 151 | Returns: |
| 152 | List of token IDs |
| 153 | """ |
| 154 | try: |
| 155 | return self._core_bpe.encode_ordinary(text) |
| 156 | except Exception as e: |
| 157 | raise TokenDaggerError(f"Encoding failed: {e}") |
| 158 | |
| 159 | def encode( |
| 160 | self, |