Function minhash_multi_process

join/LSH/new copy.py:141–165

(set_path, file_list, q, param_dic, id)

    # print("{} end,size={}".format(id,len(sizes)))

def minhash_multi_process(set_path, file_list, q, param_dic, id):
    # print("{} start".format(id))
    lsh = MinHashLSHEnsemble(threshold=param_dic['threshold'], num_perm=param_dic['num_perm'],
                             num_part=param_dic['num_part'], m=param_dic['m'], storage_config=param_dic['storage_config'])
    for i, (lower, upper) in enumerate(param_dic["partitions"]):
        lsh.lowers[i], lsh.uppers[i] = lower, upper
    for i, sets_file in enumerate(file_list):
        df = pd.read_csv(os.path.join(set_path, sets_file), dtype='str', lineterminator='\n').dropna()
        columns = df.columns.tolist()
        data = df.values.T.tolist()
        for column, vals in zip(columns, data):
            # 需要对value去重
            vals = list(set(vals))
            # 域的键值
            key = sets_file + "." + column
            # 生成minhash
            mh = MinHash(param_dic['num_perm'], hashfunc=_hash_32)
            for word in vals:
                mh.update(str(word))
            lsh.index((key, mh, len(vals)))
        if id==0:
            sys.stdout.write("\rId 0 Process Read and minhash {}/{} files".format(i+1, len(file_list)))
    if id==0:
        sys.stdout.write("\n")
    q.put(lsh)

# 将csv文件列表中每个表格中的column转换为三元组(minhashes, sets, keys)形式
# @profile