preprocess
(self, sequence)
| 134 | return self.specials.index(self.eos) |
| 135 | |
| 136 | def preprocess(self, sequence): |
| 137 | """preprocess""" |
| 138 | if self.fn is not None: |
| 139 | sequence = self.fn(sequence) |
| 140 | if self.tokenize is not None: |
| 141 | sequence = self.tokenize(sequence) |
| 142 | elif self.tokenizer is not None: |
| 143 | sequence = self.tokenizer.tokenize(sequence) |
| 144 | if not sequence: sequence = [self.unk] |
| 145 | if self.lower: |
| 146 | sequence = [token.lower() for token in sequence] |
| 147 | # If the sequence contains special characters, convert it to unk |
| 148 | sequence = [self.unk if token in self.specials else token for token in sequence] |
| 149 | return sequence |
| 150 | |
| 151 | def build(self, corpus, min_freq=1): |
| 152 | """Create vocab based on corpus""" |