Class process

datasets/ali-cpp_aitm/process_public_data.py:37–181

class process(object):
    def __init__(self):
        pass

    def process_train(self):
        c = 0
        common_feat_dict = {}
        with open(common_feat_path.format('train'), 'r') as fr:
            for line in fr:
                line_list = line.strip().split(',')
                kv = np.array(re.split('\x01|\x02|\x03', line_list[2]))
                key = kv[range(0, len(kv), 3)]
                value = kv[range(1, len(kv), 3)]
                feat_dict = dict(zip(key, value))
                common_feat_dict[line_list[0]] = feat_dict
                c += 1
                if c % 100000 == 0:
                    print(c)
        print('join feats...')
        c = 0
        vocabulary = dict(
            zip(use_columns, [{} for _ in range(len(use_columns))]))
        with open(data_path.format('train') + '.tmp', 'w') as fw:
            fw.write('click,purchase,' + ','.join(use_columns) + '\n')
            with open(data_path.format('train'), 'r') as fr:
                for line in fr:
                    line_list = line.strip().split(',')
                    if line_list[1] == '0' and line_list[2] == '1':
                        continue
                    kv = np.array(re.split('\x01|\x02|\x03', line_list[5]))
                    key = kv[range(0, len(kv), 3)]
                    value = kv[range(1, len(kv), 3)]
                    feat_dict = dict(zip(key, value))
                    feat_dict.update(common_feat_dict[line_list[3]])
                    feats = line_list[1:3]
                    for k in use_columns:
                        feats.append(feat_dict.get(k, '0'))
                    fw.write(','.join(feats) + '\n')
                    for k, v in feat_dict.items():
                        if k in use_columns:
                            if v in vocabulary[k]:
                                vocabulary[k][v] += 1
                            else:
                                vocabulary[k][v] = 0
                    c += 1
                    if c % 100000 == 0:
                        print(c)
        print('before filter low freq:')
        for k, v in vocabulary.items():
            print(k + ':' + str(len(v)))
        new_vocabulary = dict(
            zip(use_columns, [set() for _ in range(len(use_columns))]))
        for k, v in vocabulary.items():
            for k1, v1 in v.items():
                if v1 > 10:
                    new_vocabulary[k].add(k1)
        vocabulary = new_vocabulary
        print('after filter low freq:')