(self, token_id: int)
| 659 | return gguf.TokenType.CONTROL if token_id in special_ids else gguf.TokenType.NORMAL |
| 660 | |
| 661 | def get_token_score(self, token_id: int) -> float: |
| 662 | # Placeholder for actual logic to determine the token's score |
| 663 | # This needs to be implemented based on specific requirements |
| 664 | return -1000.0 # Default score |
| 665 | |
| 666 | def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: |
| 667 | for text in self.added_tokens_list: |
no outgoing calls
no test coverage detected