| 111 | self.tokenization = self.tokenization[:idx] + other.tokenization + self.tokenization[idx:] |
| 112 | |
| 113 | def append(self, other): |
| 114 | if isinstance(other, (CommandToken, TypeToken)): |
| 115 | self.tokenization.append(other.Id) |
| 116 | self.text += other.token |
| 117 | self.original_text += other.token |
| 118 | elif isinstance(other, Tokenization): |
| 119 | self.tokenization.extend(other.tokenization) |
| 120 | self.text += other.text |
| 121 | self.original_text += other.original_text |
| 122 | else: |
| 123 | self.tokenization.append(other) |
| 124 | return self |
| 125 | |
| 126 | def extend(self, other): |
| 127 | if isinstance(other, (CommandToken, TypeToken)): |