ascii character to index
(self, token)
| 595 | return chr(Id) |
| 596 | |
| 597 | def TokenToId(self, token): |
| 598 | """ascii character to index""" |
| 599 | return ord(token) |
| 600 | |
| 601 | def DecodeIds(self, Ids): |
| 602 | """converts ascii ids to tokens before joining them into text""" |