Function limit_encode_train

examples/qabot/qabot_data.py:114–165

prepare train data to embedded word vector sequence given sequence limit return: questions_encoded: np ndarray, shape (number samples, seq length, vector size) poss_encoded: same layout, sequence for positive answer negs_encoded: same layo

(train_raw, label2answer, idx2word, q_seq_limit,
                       ans_seq_limit, idx2vec)

def limit_encode_train(train_raw, label2answer, idx2word, q_seq_limit,
                       ans_seq_limit, idx2vec):
    ''' prepare train data to embedded word vector sequence given sequence limit
        return:
            questions_encoded: np ndarray, shape
                (number samples, seq length, vector size)
            poss_encoded: same layout, sequence for positive answer
            negs_encoded: same layout, sequence for negative answer
    '''
    questions = [question for question, answers, candis in train_raw]
    # choose 1 answer from answer pool
    poss = [
        label2answer[random.choice(answers)]
        for question, answers, candis in train_raw
    ]
    # choose 1 candidate from candidate pool
    negs = [
        label2answer[random.choice(candis)]
        for question, answers, candis in train_raw
    ]

    # filtered word not in idx2vec
    questions_filtered = [
        [idx for idx in q if idx in idx2vec] for q in questions
    ]
    poss_filtered = [[idx for idx in ans if idx in idx2vec] for ans in poss]
    negs_filtered = [[idx for idx in ans if idx in idx2vec] for ans in negs]

    # crop to seq limit
    questions_crop = [
        q[:q_seq_limit] + [0] * max(0, q_seq_limit - len(q))
        for q in questions_filtered
    ]
    poss_crop = [
        ans[:ans_seq_limit] + [0] * max(0, ans_seq_limit - len(ans))
        for ans in poss_filtered
    ]
    negs_crop = [
        ans[:ans_seq_limit] + [0] * max(0, ans_seq_limit - len(ans))
        for ans in negs_filtered
    ]

    # encoded, word idx to word vector
    questions_encoded = [[idx2vec[idx] for idx in q] for q in questions_crop]
    poss_encoded = [[idx2vec[idx] for idx in ans] for ans in poss_crop]
    negs_encoded = [[idx2vec[idx] for idx in ans] for ans in negs_crop]

    # make nd array
    questions_encoded = np.array(questions_encoded).astype(np.float32)
    poss_encoded = np.array(poss_encoded).astype(np.float32)
    negs_encoded = np.array(negs_encoded).astype(np.float32)
    return questions_encoded, poss_encoded, negs_encoded


def get_idx2vec_weights(wv, idx2word):