converts ids to wordpiece tokens and joins them as a text string
(self, Ids, type_token=False)
| 902 | return self.text_tokenizer.vocab[token] |
| 903 | |
| 904 | def DecodeIds(self, Ids, type_token=False): |
| 905 | """converts ids to wordpiece tokens and joins them as a text string""" |
| 906 | if type_token: |
| 907 | return ' '.join(Id.token if isinstance(Id, TypeToken) else self.type_id_map[Id].token for Id in Ids) |
| 908 | if isinstance(Ids, Tokenization): |
| 909 | Ids = Ids.tokenization |
| 910 | Tokens = [] |
| 911 | for Id in Ids: |
| 912 | if Id in self.command_id_map: |
| 913 | Tokens.append(self.command_id_map[Id].token) |
| 914 | elif Id in self.text_tokenizer.ids_to_tokens: |
| 915 | Tokens.append(self.text_tokenizer.ids_to_tokens[Id]) |
| 916 | new_tokens = [] |
| 917 | for token in Tokens: |
| 918 | if token.startswith('##') and len(new_tokens) > 0: |
| 919 | new_tokens[-1] += token[2:] |
| 920 | else: |
| 921 | new_tokens.append(token) |
| 922 | return ' '.join(new_tokens) |
| 923 | |
| 924 | def DecodeTokens(self, Tokens, type_token=False): |
| 925 | """converts wordpiece tokens to a text string""" |