Tokenizer object that handles text tokenization, command tokens, and type tokens. Command tokens and text tokens are stored together in one mapping of size `len(text_tokenizer)+len(command_tokens)`. Command tokens are stored as first `len(command_tokens)` tokens. Token idx is
| 214 | |
| 215 | |
| 216 | class Tokenizer(object): |
| 217 | """ |
| 218 | Tokenizer object that handles text tokenization, command tokens, and type tokens. |
| 219 | |
| 220 | Command tokens and text tokens are stored together in one mapping of size |
| 221 | `len(text_tokenizer)+len(command_tokens)`. Command tokens are stored as first |
| 222 | `len(command_tokens)` tokens. Token idx is stored at `idx+len(command_tokens)`. |
| 223 | |
| 224 | Token types are stored in a separate mapping of size `len(type_tokens)`. |
| 225 | """ |
| 226 | |
| 227 | def __init__(self, text_tokenizer, command_tokens=None, type_tokens=None): |
| 228 | # set text tokenizer |
| 229 | self.text_tokenizer = text_tokenizer |
| 230 | if not hasattr(self, 'num_text_tokens'): |
| 231 | self.num_text_tokens = len(self.text_tokenizer) |
| 232 | |
| 233 | # set command tokens |
| 234 | if command_tokens is None: |
| 235 | command_tokens = DEFAULT_COMMAND_TOKENS |
| 236 | self._command_tokens = command_tokens |
| 237 | self.command_name_map = {tok.name: tok for tok in self._command_tokens} |
| 238 | self.command_token_map = {tok.token: tok for tok in self._command_tokens} |
| 239 | self.command_id_map = {tok.Id: tok for tok in self._command_tokens} |
| 240 | if not hasattr(self, 'num_command_tokens'): |
| 241 | self.num_command_tokens = len(self._command_tokens) |
| 242 | if not hasattr(self, 'num_tokens'): |
| 243 | self.num_tokens = self.num_command_tokens + self.num_text_tokens |
| 244 | |
| 245 | # set type tokens |
| 246 | if type_tokens is None: |
| 247 | type_tokens = DEFAULT_TYPE_TOKENS |
| 248 | self.type_tokens = type_tokens |
| 249 | self.type_name_map = {tok.name: tok for tok in self.type_tokens} |
| 250 | self.type_token_map = {tok.token: tok for tok in self.type_tokens} |
| 251 | self.type_id_map = {tok.Id: tok for tok in self.type_tokens} |
| 252 | if not hasattr(self, 'num_type_tokens'): |
| 253 | self.num_type_tokens = len(self.type_tokens) |
| 254 | |
| 255 | # parse tokens and vocabs from tokenizer |
| 256 | self._tokens = list(self.command_token_map.keys()) + list(self.text_tokenizer.tokens) |
| 257 | self._vocab = {t: Id for Id, t in self.command_id_map.items()} |
| 258 | self._vocab.update({t: Id + self.num_command_tokens for t, Id in self.text_tokenizer.vocab.items()}) |
| 259 | |
| 260 | self._text_tokens = list(self.text_tokenizer.tokens) |
| 261 | self._text_token_vocab = {t: Id + self.num_command_tokens for t, Id in self.text_tokenizer.vocab.items()} |
| 262 | |
| 263 | self._command_token_tokens = list(self.command_token_map.keys()) |
| 264 | self._command_token_vocab = {t: Id for Id, t in self.command_id_map.items()} |
| 265 | |
| 266 | self._token_types = list(self.type_token_map.keys()) |
| 267 | self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()} |
| 268 | |
| 269 | def __call__(self, text, process_fn=None): |
| 270 | """run preprocessing and encode text as Ids""" |
| 271 | return self.EncodeAsIds(text, process_fn=process_fn) |
| 272 | |
| 273 | def __len__(self): |