Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.
(self, sequences)
| 211 | super(ErnieField, self).__init__(*args, **kwargs) |
| 212 | |
| 213 | def transform(self, sequences): |
| 214 | """Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.""" |
| 215 | sequences = [[self.preprocess(token) for token in seq] for seq in sequences] |
| 216 | if self.fix_len <= 0: |
| 217 | self.fix_len = max(len(token) for seq in sequences for token in seq) |
| 218 | if self.use_vocab: |
| 219 | sequences = [[[self.vocab[i] for i in token] for token in seq] for seq in sequences] |
| 220 | if self.bos: |
| 221 | sequences = [[[self.bos_index]] + seq for seq in sequences] |
| 222 | if self.eos: |
| 223 | sequences = [seq + [[self.eos_index]] for seq in sequences] |
| 224 | sequences = [ |
| 225 | nn.pad_sequence([np.array(ids[:self.fix_len], dtype=np.int64) for ids in seq], self.pad_index, self.fix_len) |
| 226 | for seq in sequences |
| 227 | ] |
| 228 | |
| 229 | return sequences |
nothing calls this directly
no test coverage detected