Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.
(self, sequences)
| 157 | self.vocab = Vocab(counter, min_freq, self.specials, self.unk_index) |
| 158 | |
| 159 | def transform(self, sequences): |
| 160 | """Sequences transform function, such as converting word to id, adding bos tags to sequences, etc.""" |
| 161 | sequences = [self.preprocess(seq) for seq in sequences] |
| 162 | if self.use_vocab: |
| 163 | sequences = [self.vocab[seq] for seq in sequences] |
| 164 | if self.bos: |
| 165 | sequences = [[self.bos_index] + seq for seq in sequences] |
| 166 | if self.eos: |
| 167 | sequences = [seq + [self.eos_index] for seq in sequences] |
| 168 | |
| 169 | sequences = [np.array(seq, dtype=np.int64) for seq in sequences] |
| 170 | |
| 171 | return sequences |
| 172 | |
| 173 | |
| 174 | class SubwordField(Field): |