A dummy tokenizer that encodes each character as its ASCII code.
| 22 | |
| 23 | |
| 24 | class MockTokenizer: |
| 25 | """A dummy tokenizer that encodes each character as its ASCII code.""" |
| 26 | |
| 27 | bos_id = 1 |
| 28 | eos_id = 2 |
| 29 | |
| 30 | def encode( |
| 31 | self, |
| 32 | string: str, |
| 33 | device: Optional[torch.device] = None, |
| 34 | bos: bool = False, |
| 35 | eos: bool = False, |
| 36 | max_length=-1, |
| 37 | ) -> torch.Tensor: |
| 38 | tokens = [ord(c) for c in string] |
| 39 | if bos: |
| 40 | bos_id = self.bos_id |
| 41 | if bos_id is None: |
| 42 | raise NotImplementedError( |
| 43 | 'Attempted to add bos, but this tokenizer does not defined a bos token.' |
| 44 | ) |
| 45 | tokens = [bos_id] + tokens |
| 46 | |
| 47 | if max_length > 0: |
| 48 | if eos: |
| 49 | if self.eos_id is None: |
| 50 | raise NotImplementedError( |
| 51 | 'Attempted to add eos, but this tokenizer does not defined an eos token' |
| 52 | ) |
| 53 | if len(tokens) >= max_length: |
| 54 | tokens = tokens[:max_length] |
| 55 | tokens[-1] = self.eos_id |
| 56 | else: |
| 57 | tokens = tokens + [self.eos_id] |
| 58 | else: |
| 59 | if eos: |
| 60 | if self.eos_id is None: |
| 61 | raise NotImplementedError( |
| 62 | 'Attempted to add eos, but this tokenizer does not defined an eos token' |
| 63 | ) |
| 64 | tokens = tokens + [self.eos_id] |
| 65 | |
| 66 | return torch.tensor(tokens) |
| 67 | |
| 68 | def decode(self, tokens: torch.Tensor) -> str: |
| 69 | decoded = '' |
| 70 | for token in tokens.tolist(): |
| 71 | if token == -100: |
| 72 | decoded += '<ignore_index>' |
| 73 | continue |
| 74 | |
| 75 | if token == 1: |
| 76 | decoded += '<s>' |
| 77 | continue |
| 78 | |
| 79 | if token == 2: |
| 80 | decoded += '</s>' |
| 81 | continue |
no outgoing calls