| 22 | |
| 23 | |
| 24 | def prepare_data(input, target, maxlen=None, return_neg=False): |
| 25 | # x: a list of sentences |
| 26 | lengths_x = [len(s[4]) for s in input] |
| 27 | seqs_mid = [inp[3] for inp in input] |
| 28 | seqs_cat = [inp[4] for inp in input] |
| 29 | noclk_seqs_mid = [inp[5] for inp in input] |
| 30 | noclk_seqs_cat = [inp[6] for inp in input] |
| 31 | |
| 32 | if maxlen is not None: |
| 33 | new_seqs_mid = [] |
| 34 | new_seqs_cat = [] |
| 35 | new_noclk_seqs_mid = [] |
| 36 | new_noclk_seqs_cat = [] |
| 37 | new_lengths_x = [] |
| 38 | for l_x, inp in zip(lengths_x, input): |
| 39 | if l_x > maxlen: |
| 40 | new_seqs_mid.append(inp[3][l_x - maxlen:]) |
| 41 | new_seqs_cat.append(inp[4][l_x - maxlen:]) |
| 42 | new_noclk_seqs_mid.append(inp[5][l_x - maxlen:]) |
| 43 | new_noclk_seqs_cat.append(inp[6][l_x - maxlen:]) |
| 44 | new_lengths_x.append(maxlen) |
| 45 | else: |
| 46 | new_seqs_mid.append(inp[3]) |
| 47 | new_seqs_cat.append(inp[4]) |
| 48 | new_noclk_seqs_mid.append(inp[5]) |
| 49 | new_noclk_seqs_cat.append(inp[6]) |
| 50 | new_lengths_x.append(l_x) |
| 51 | lengths_x = new_lengths_x |
| 52 | seqs_mid = new_seqs_mid |
| 53 | seqs_cat = new_seqs_cat |
| 54 | noclk_seqs_mid = new_noclk_seqs_mid |
| 55 | noclk_seqs_cat = new_noclk_seqs_cat |
| 56 | |
| 57 | if len(lengths_x) < 1: |
| 58 | return None, None, None, None |
| 59 | |
| 60 | n_samples = len(seqs_mid) |
| 61 | maxlen_x = numpy.max(lengths_x) |
| 62 | neg_samples = len(noclk_seqs_mid[0][0]) |
| 63 | |
| 64 | mid_his = numpy.zeros((n_samples, maxlen_x)).astype('int64') |
| 65 | cat_his = numpy.zeros((n_samples, maxlen_x)).astype('int64') |
| 66 | noclk_mid_his = numpy.zeros( |
| 67 | (n_samples, maxlen_x, neg_samples)).astype('int64') |
| 68 | noclk_cat_his = numpy.zeros( |
| 69 | (n_samples, maxlen_x, neg_samples)).astype('int64') |
| 70 | mid_mask = numpy.zeros((n_samples, maxlen_x)).astype('float32') |
| 71 | for idx, [s_x, s_y, no_sx, no_sy] in enumerate( |
| 72 | zip(seqs_mid, seqs_cat, noclk_seqs_mid, noclk_seqs_cat)): |
| 73 | mid_mask[idx, :lengths_x[idx]] = 1. |
| 74 | mid_his[idx, :lengths_x[idx]] = s_x |
| 75 | cat_his[idx, :lengths_x[idx]] = s_y |
| 76 | noclk_mid_his[idx, :lengths_x[idx], :] = no_sx |
| 77 | noclk_cat_his[idx, :lengths_x[idx], :] = no_sy |
| 78 | |
| 79 | uids = numpy.array([inp[0] for inp in input]) |
| 80 | mids = numpy.array([inp[1] for inp in input]) |
| 81 | cats = numpy.array([inp[2] for inp in input]) |