Create vocab based on corpus
(self, corpus, min_freq=1)
| 149 | return sequence |
| 150 | |
| 151 | def build(self, corpus, min_freq=1): |
| 152 | """Create vocab based on corpus""" |
| 153 | if hasattr(self, 'vocab'): |
| 154 | return |
| 155 | sequences = getattr(corpus, self.name) |
| 156 | counter = Counter(token for seq in sequences for token in self.preprocess(seq)) |
| 157 | self.vocab = Vocab(counter, min_freq, self.specials, self.unk_index) |
| 158 | |
| 159 | def transform(self, sequences): |
| 160 | """Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.""" |
no test coverage detected