(self, s: str, bos: bool = False, eos: bool = False)
| 48 | return self.sp_model.EncodeAsPieces(s) |
| 49 | |
| 50 | def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: |
| 51 | assert type(s) is str |
| 52 | t = self.sp_model.encode(s) |
| 53 | if bos: |
| 54 | t = [self.bos_id] + t |
| 55 | if eos: |
| 56 | t = t + [self.eos_id] |
| 57 | return t |
| 58 | |
| 59 | def decode(self, t: List[int]) -> str: |
| 60 | text, buffer = "", [] |
no outgoing calls
no test coverage detected