Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.
(self, sequences)
| 186 | self.vocab = Vocab(counter, min_freq, self.specials, self.unk_index) |
| 187 | |
| 188 | def transform(self, sequences): |
| 189 | """Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.""" |
| 190 | sequences = [[self.preprocess(token) for token in seq] for seq in sequences] |
| 191 | if self.fix_len <= 0: |
| 192 | self.fix_len = max(len(token) for seq in sequences for token in seq) |
| 193 | if self.use_vocab: |
| 194 | sequences = [[[self.vocab[i] for i in token] for token in seq] for seq in sequences] |
| 195 | if self.bos: |
| 196 | sequences = [[[self.bos_index]] + seq for seq in sequences] |
| 197 | if self.eos: |
| 198 | sequences = [seq + [[self.eos_index]] for seq in sequences] |
| 199 | sequences = [ |
| 200 | nn.pad_sequence([np.array(ids[:self.fix_len], dtype=np.int64) for ids in seq], self.pad_index, self.fix_len) |
| 201 | for seq in sequences |
| 202 | ] |
| 203 | |
| 204 | return sequences |
| 205 | |
| 206 | |
| 207 | class ErnieField(Field): |
nothing calls this directly
no test coverage detected