Tokenization object to hold tokenization, (processed text),and original text. Can hold tokenization as Ids or tokens. It also holds command tokens (pad, unk, etc.) for the tokenization. This allows functions to pad/operate on tokenizations without having access to the ful
| 54 | |
| 55 | |
| 56 | class Tokenization(object): |
| 57 | """ |
| 58 | Tokenization object to hold tokenization, (processed text),and original |
| 59 | text. Can hold tokenization as Ids or tokens. |
| 60 | |
| 61 | It also holds command tokens (pad, unk, etc.) for the tokenization. |
| 62 | This allows functions to pad/operate on tokenizations without having |
| 63 | access to the full tokenizer, just the tokenization. |
| 64 | |
| 65 | Several standard array operations are implemented (insert, append, extend). |
| 66 | """ |
| 67 | |
| 68 | def __init__(self, tokenization, text=None, original_text=None, command_tokens=None, asIds=True): |
| 69 | self.tokenization = tokenization |
| 70 | self.text = text |
| 71 | if self.text is None: |
| 72 | self.text = self.tokenization |
| 73 | self.original_text = original_text |
| 74 | if self.original_text is None: |
| 75 | self.original_text = self.text |
| 76 | self.command_tokens = command_tokens |
| 77 | self.asIds = asIds |
| 78 | self.parse_command_tokens() |
| 79 | |
| 80 | def set_command_tokens(self, command_tokens): |
| 81 | self.command_tokens = command_tokens |
| 82 | return self.parse_command_tokens() |
| 83 | |
| 84 | def parse_command_tokens(self): |
| 85 | if self.command_tokens is None: |
| 86 | return |
| 87 | for command_token in self.command_tokens: |
| 88 | if self.asIds: |
| 89 | setattr(self, command_token.name, command_token.Id) |
| 90 | else: |
| 91 | setattr(self, command_token.name, command_token.token) |
| 92 | |
| 93 | def __getitem__(self, index): |
| 94 | return self.tokenization[index] |
| 95 | |
| 96 | def __len__(self): |
| 97 | return len(self.tokenization) |
| 98 | |
| 99 | def insert(self, idx, other): |
| 100 | if isinstance(other, (CommandToken, TypeToken)): |
| 101 | self.tokenization.insert(idx, other.Id) |
| 102 | if idx == 0: |
| 103 | self.text = other.token + self.text |
| 104 | self.original_text = other.token + self.original_text |
| 105 | elif idx == len(self.tokenization) - 1: |
| 106 | self.text += other.token |
| 107 | self.original_text += other.token |
| 108 | elif isinstance(other, Tokenization): |
| 109 | self.tokenization = self.tokenization[:idx] + other.tokenization + self.tokenization[idx:] |
| 110 | else: |
| 111 | self.tokenization = self.tokenization[:idx] + other.tokenization + self.tokenization[idx:] |
| 112 | |
| 113 | def append(self, other): |
no outgoing calls
no test coverage detected