Tokenization object to hold tokenization, (processed text),and original text. Can hold tokenization as Ids or tokens. It also holds command tokens (pad, unk, etc.) for the tokenization. This allows functions to pad/operate on tokenization without having access to the full token
| 27 | from .sp_tokenizer import SentencePieceTokenizer |
| 28 | |
| 29 | class Tokenization(object): |
| 30 | """ |
| 31 | Tokenization object to hold tokenization, (processed text),and original |
| 32 | text. Can hold tokenization as Ids or tokens. |
| 33 | |
| 34 | It also holds command tokens (pad, unk, etc.) for the tokenization. |
| 35 | This allows functions to pad/operate on tokenization without having |
| 36 | access to the full tokenizer, just the tokenization. |
| 37 | |
| 38 | Several standard array operations are implemented (insert, append, extend). |
| 39 | """ |
| 40 | |
| 41 | def __init__(self, tokenization, text=None, original_text=None, command_tokens=None, asIds=True): |
| 42 | self.tokenization = tokenization |
| 43 | self.text = text |
| 44 | if self.text is None: |
| 45 | self.text = self.tokenization |
| 46 | self.original_text = original_text |
| 47 | if self.original_text is None: |
| 48 | self.original_text = self.text |
| 49 | self.command_tokens = command_tokens |
| 50 | self.asIds = asIds |
| 51 | self.parse_command_tokens() |
| 52 | |
| 53 | def set_command_tokens(self, command_tokens): |
| 54 | self.command_tokens = command_tokens |
| 55 | return self.parse_command_tokens() |
| 56 | |
| 57 | def parse_command_tokens(self): |
| 58 | if self.command_tokens is None: |
| 59 | return |
| 60 | for command_token in self.command_tokens: |
| 61 | if self.asIds: |
| 62 | setattr(self, command_token.name, command_token.Id) |
| 63 | else: |
| 64 | setattr(self, command_token.name, command_token.token) |
| 65 | |
| 66 | def __getitem__(self, index): |
| 67 | return self.tokenization[index] |
| 68 | |
| 69 | def __len__(self): |
| 70 | return len(self.tokenization) |
| 71 | |
| 72 | def __str__(self): |
| 73 | return f"Tokenization = {self.tokenization}, Text = {self.text}" |
| 74 | |
| 75 | def insert(self, idx, other): |
| 76 | if isinstance(other, CommandToken): |
| 77 | self.tokenization.insert(idx, other.Id) |
| 78 | if idx == 0: |
| 79 | self.text = other.token + self.text |
| 80 | self.original_text = other.token + self.original_text |
| 81 | elif idx == len(self.tokenization) - 1: |
| 82 | self.text += other.token |
| 83 | self.original_text += other.token |
| 84 | elif isinstance(other, Tokenization): |
| 85 | self.tokenization = self.tokenization[:idx] + other.tokenization + self.tokenization[idx:] |
| 86 | else: |
no outgoing calls
no test coverage detected