convert tokens to a string accounting for command and type tokens.
(self, Tokens, type_token=False)
| 451 | return ' '.join(rtn_strs) |
| 452 | |
| 453 | def DecodeTokens(self, Tokens, type_token=False): |
| 454 | """ |
| 455 | convert tokens to a string accounting for command and type tokens. |
| 456 | """ |
| 457 | if type_token: |
| 458 | return ' '.join(t.token if isinstance(t, TypeToken) else t for t in Tokens) |
| 459 | rtn_strs = [] |
| 460 | current_str = [] |
| 461 | if isinstance(Tokens, Tokenization): |
| 462 | Tokens = Tokens.tokenization |
| 463 | for t in Tokens: |
| 464 | if isinstance(t, CommandToken): |
| 465 | rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) |
| 466 | current_str = [] |
| 467 | rtn_strs.append(t.token) |
| 468 | elif t in self.command_token_map: |
| 469 | rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) |
| 470 | current_str = [] |
| 471 | rtn_strs.append(t) |
| 472 | else: |
| 473 | current_str.append(t) |
| 474 | if current_str != []: |
| 475 | rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) |
| 476 | return ' '.join(rtn_strs) |
| 477 | |
| 478 | |
| 479 | class TextTokenizer(object): |