(self, tokens_list: list[list[int]])
| 155 | |
| 156 | # +++ 新增 +++: 增加一个批量解码的方法以提高效率,并保持接口一致性 |
| 157 | def decode_batch(self, tokens_list: list[list[int]]) -> list[str]: |
| 158 | self._lazy_load_tokenizer() |
| 159 | # HuggingFace tokenizer 有 decode_batch,但 tiktoken 没有,我们用列表推导来模拟 |
| 160 | if self.tokenizer_type == "tiktoken": |
| 161 | return [self._tokenizer.decode(tokens) for tokens in tokens_list] |
| 162 | elif self.tokenizer_type == "huggingface": |
| 163 | return self._tokenizer.batch_decode(tokens_list, skip_special_tokens=True) |
| 164 | else: |
| 165 | raise ValueError(f"Unknown tokenizer_type: {self.tokenizer_type}") |
| 166 | |
| 167 | |
| 168 |
no test coverage detected