| 1135 | |
| 1136 | |
| 1137 | class ChineseSPTokenizer(Tokenizer): |
| 1138 | def __init__(self, add_block_symbols=False, add_task_mask=False, add_decoder_mask=False, fix_command_token=False, |
| 1139 | **kwargs): |
| 1140 | self.text_tokenizer = sp_tokenizer.from_pretrained() |
| 1141 | |
| 1142 | self.num_command_tokens = 0 |
| 1143 | self.num_text_tokens = self.text_tokenizer.sp.vocab_size() |
| 1144 | self.num_tokens = self.num_text_tokens |
| 1145 | self.num_type_tokens = 2 |
| 1146 | |
| 1147 | self._command_tokens = [ |
| 1148 | CommandToken('pad', '<|endoftext|>', self.num_text_tokens), |
| 1149 | CommandToken('eos', '<|endoftext|>', self.num_text_tokens), |
| 1150 | CommandToken('sep', '[SEP]', self.num_text_tokens + 1), |
| 1151 | CommandToken('ENC', '[CLS]', self.num_text_tokens + 2), |
| 1152 | CommandToken('MASK', '[MASK]', self.num_text_tokens + 3, lstrip=True), |
| 1153 | CommandToken('unk', '[UNK]', self.num_text_tokens + 4) |
| 1154 | ] |
| 1155 | self.num_tokens += 5 |
| 1156 | self.num_command_tokens += 6 |
| 1157 | if add_block_symbols: |
| 1158 | self._command_tokens.extend([ |
| 1159 | CommandToken('sop', '<|startofpiece|>', self.num_tokens + 1), |
| 1160 | CommandToken('eop', '<|endofpiece|>', self.num_tokens + 2) |
| 1161 | ]) |
| 1162 | if fix_command_token: |
| 1163 | self.num_tokens += 3 |
| 1164 | else: |
| 1165 | self.num_tokens += 2 |
| 1166 | self.num_command_tokens += 2 |
| 1167 | if add_task_mask: |
| 1168 | if fix_command_token: |
| 1169 | self._command_tokens.extend([ |
| 1170 | CommandToken('sMASK', '[sMASK]', self.num_tokens, lstrip=True), |
| 1171 | CommandToken('gMASK', '[gMASK]', self.num_tokens + 1, lstrip=True) |
| 1172 | ]) |
| 1173 | else: |
| 1174 | self._command_tokens.extend([ |
| 1175 | CommandToken('gMASK', '[gMASK]', self.num_tokens, lstrip=True), |
| 1176 | CommandToken('sMASK', '[sMASK]', self.num_tokens + 1, lstrip=True) |
| 1177 | ]) |
| 1178 | self.num_tokens += 2 |
| 1179 | self.num_command_tokens += 2 |
| 1180 | if add_decoder_mask: |
| 1181 | self._command_tokens.extend([ |
| 1182 | CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens) |
| 1183 | ]) |
| 1184 | self.num_tokens += 1 |
| 1185 | self.num_command_tokens += 1 |
| 1186 | self.command_name_map = {tok.name: tok for tok in self._command_tokens} |
| 1187 | self.command_token_map = {tok.token: tok for tok in self._command_tokens} |
| 1188 | self.command_id_map = {tok.Id: tok for tok in self._command_tokens} |
| 1189 | print_rank_0({tok.name: tok.Id for tok in self._command_tokens}) |
| 1190 | self.type_tokens = [ |
| 1191 | TypeToken('str0', '<str0>', 0), |
| 1192 | TypeToken('str1', '<str1>', 1), |
| 1193 | ] |
| 1194 | self.type_name_map = {tok.name: tok for tok in self.type_tokens} |