converts ascii ids to tokens before joining them into text
(self, Ids)
| 599 | return ord(token) |
| 600 | |
| 601 | def DecodeIds(self, Ids): |
| 602 | """converts ascii ids to tokens before joining them into text""" |
| 603 | if isinstance(Ids, Tokenization): |
| 604 | Ids = Ids.tokenization |
| 605 | return ''.join([self.IdToToken(tok) for tok in Ids]) |
| 606 | |
| 607 | def DecodeTokens(self, Tokens): |
| 608 | """just concatenates ascii tokens into text""" |