Create vocab based on corpus
(self, corpus, min_freq=1)
| 178 | super(SubwordField, self).__init__(*args, **kwargs) |
| 179 | |
| 180 | def build(self, corpus, min_freq=1): |
| 181 | """Create vocab based on corpus""" |
| 182 | if hasattr(self, 'vocab'): |
| 183 | return |
| 184 | sequences = getattr(corpus, self.name) |
| 185 | counter = Counter(piece for seq in sequences for token in seq for piece in self.preprocess(token)) |
| 186 | self.vocab = Vocab(counter, min_freq, self.specials, self.unk_index) |
| 187 | |
| 188 | def transform(self, sequences): |
| 189 | """Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.""" |
nothing calls this directly
no test coverage detected