converts wordpiece tokens to a text string
(self, Tokens, type_token=False)
| 922 | return ' '.join(new_tokens) |
| 923 | |
| 924 | def DecodeTokens(self, Tokens, type_token=False): |
| 925 | """converts wordpiece tokens to a text string""" |
| 926 | if type_token: |
| 927 | return ' '.join(t.token if isinstance(t, TypeToken) else t for t in Tokens) |
| 928 | if isinstance(Tokens, Tokenization): |
| 929 | Tokens = Tokens.tokenization |
| 930 | return ' '.join(Tokens) |
| 931 | |
| 932 | |
| 933 | class GPT2BPETokenizer(Tokenizer): |
nothing calls this directly
no outgoing calls
no test coverage detected