(self, other)
| 113 | return self |
| 114 | |
| 115 | def extend(self, other): |
| 116 | if isinstance(other, (CommandToken, TypeToken)): |
| 117 | self.tokenization.append(other.Id) |
| 118 | self.text += other.token |
| 119 | self.original_text += other.token |
| 120 | elif isinstance(other, list) and isinstance(other[0], (CommandToken, TypeToken)): |
| 121 | self.tokenization.extend([o.Id for o in other]) |
| 122 | self.text += [o.token for o in other] |
| 123 | self.original_text += [o.token for o in other] |
| 124 | elif isinstance(other, Tokenization): |
| 125 | self.tokenization.extend(other.tokenization) |
| 126 | self.text += other.text |
| 127 | self.original_text += other.original_text |
| 128 | else: |
| 129 | self.tokenization.extend(other) |
| 130 | return self |
| 131 | |
| 132 | """define some default command tokens for the tokenizer to use""" |
| 133 | token_format = "<{0}>" |
no test coverage detected