Function limit_encode_eval

examples/qabot/qabot_data.py:212–282

prepare train data to embedded word vector sequence given sequence limit for testing return: questions_encoded: np ndarray, shape (number samples, seq length, vector size) poss_encoded: same layout, sequence for positive answer negs_encode

(train_raw,
                      label2answer,
                      idx2word,
                      q_seq_limit,
                      ans_seq_limit,
                      idx2vec,
                      top_k_candi_limit=6)

def limit_encode_eval(train_raw,
                      label2answer,
                      idx2word,
                      q_seq_limit,
                      ans_seq_limit,
                      idx2vec,
                      top_k_candi_limit=6):
    ''' prepare train data to embedded word vector sequence given sequence limit for testing
        return:
            questions_encoded: np ndarray, shape
                (number samples, seq length, vector size)
            poss_encoded: same layout, sequence for positive answer
            negs_encoded: same layout, sequence for negative answer
    '''
    questions = [question for question, answers, candis in train_raw]

    # combine truth and candidate answers label,
    candi_pools = [
        list(answers + candis)[:top_k_candi_limit]
        for question, answers, candis in train_raw
    ]
    assert all([len(pool) == top_k_candi_limit for pool in candi_pools])

    ans_count = [len(answers) for question, answers, candis in train_raw]
    assert all([c > 0 for c in ans_count])

    # encode ans
    candi_pools_encoded = [[label2answer[candi_label]
                            for candi_label in pool]
                           for pool in candi_pools]

    # filtered word not in idx2vec
    questions_filtered = [
        [idx for idx in q if idx in idx2vec] for q in questions
    ]
    candi_pools_filtered = [[[idx
                              for idx in candi_encoded
                              if idx in idx2vec]
                             for candi_encoded in pool]
                            for pool in candi_pools_encoded]

    # crop to seq limit
    questions_crop = [
        q[:q_seq_limit] + [0] * max(0, q_seq_limit - len(q))
        for q in questions_filtered
    ]
    candi_pools_crop = [[
        candi[:ans_seq_limit] + [0] * max(0, ans_seq_limit - len(candi))
        for candi in pool
    ]
                        for pool in candi_pools_filtered]

    # encoded, word idx to word vector
    questions_encoded = [[idx2vec[idx] for idx in q] for q in questions_crop]
    candi_pools_encoded = [[[idx2vec[idx]
                             for idx in candi]
                            for candi in pool]
                           for pool in candi_pools_crop]