Create the matrices from the datasets. This pad each sequence to the same lenght: the lenght of the longuest sequence or maxlen. if maxlen is set, we will cut all sequence to this maximum lenght. This swap the axis!
(seqs, labels, maxlen=None)
| 10 | |
| 11 | |
| 12 | def prepare_data(seqs, labels, maxlen=None): |
| 13 | """Create the matrices from the datasets. |
| 14 | |
| 15 | This pad each sequence to the same lenght: the lenght of the |
| 16 | longuest sequence or maxlen. |
| 17 | |
| 18 | if maxlen is set, we will cut all sequence to this maximum |
| 19 | lenght. |
| 20 | |
| 21 | This swap the axis! |
| 22 | """ |
| 23 | # x: a list of sentences |
| 24 | lengths = [len(s) for s in seqs] |
| 25 | |
| 26 | if maxlen is not None: |
| 27 | new_seqs = [] |
| 28 | new_labels = [] |
| 29 | new_lengths = [] |
| 30 | for l, s, y in zip(lengths, seqs, labels): |
| 31 | if l < maxlen: |
| 32 | new_seqs.append(s) |
| 33 | new_labels.append(y) |
| 34 | new_lengths.append(l) |
| 35 | lengths = new_lengths |
| 36 | labels = new_labels |
| 37 | seqs = new_seqs |
| 38 | |
| 39 | if len(lengths) < 1: |
| 40 | return None, None, None |
| 41 | |
| 42 | n_samples = len(seqs) |
| 43 | maxlen = numpy.max(lengths) |
| 44 | |
| 45 | x = numpy.zeros((maxlen, n_samples)).astype('int64') |
| 46 | x_mask = numpy.zeros((maxlen, n_samples)).astype(theano.config.floatX) |
| 47 | for idx, s in enumerate(seqs): |
| 48 | x[:lengths[idx], idx] = s |
| 49 | x_mask[:lengths[idx], idx] = 1. |
| 50 | |
| 51 | return x, x_mask, labels |
| 52 | |
| 53 | |
| 54 | def get_dataset_file(dataset, default_dataset, origin): |
no outgoing calls
no test coverage detected