Function get_train_raw

examples/qabot/qabot_data.py:86–111

deserialize training data file args: data_dir: dir of data file return: train_raw: list of QnA pair, length of list == number of samples, each pair has 3 fields: 0 is question sentence idx encoded, use idx2word to decode,

(data_dir, data_filename)

def get_train_raw(data_dir, data_filename):
    ''' deserialize training data file
        args:
            data_dir: dir of data file
        return:
            train_raw: list of QnA pair, length of list  == number of samples,
                each pair has 3 fields:
                    0 is question sentence idx encoded, use idx2word to decode,
                        idx2vec to get embedding.
                    1 is ans labels, each label corresponds to a ans sentence,
                        use label2answer to decode.
                    2 is top K candidate ans, these are negative ans for
                        training.
    '''
    train_raw = []
    import gzip
    with gzip.open(data_dir + data_filename) as fin:
        for line in fin:
            tpl = line.decode().strip().split("\t")
            question = [
                int(idx.replace("idx_", "")) for idx in tpl[1].split(" ")
            ]
            ans = [int(label) for label in tpl[2].split(" ")]
            candis = [int(label) for label in tpl[3].split(" ")]
            train_raw.append((question, ans, candis))
    return train_raw


def limit_encode_train(train_raw, label2answer, idx2word, q_seq_limit,