Function minhash_multi_process

union/TUS/Uset.py:121–155

(folder_path, file_list, q, spl, id, n, type, t)

    return result

def minhash_multi_process(folder_path, file_list, q, spl, id, n, type, t):
    # print("{} start".format(id))
    lsh = MinHashLSH(threshold=t, num_perm=128)

    for i, filename in enumerate(file_list):
        if filename.endswith('.csv'):
            #file_path = os.path.join(folder_path, filename)
            df = pd.read_csv(filename, lineterminator='\n', low_memory=False)
            #取出每一列，将每一列的值minhah，然后加入lsh中，创建索引，列的索引用文件名+列名
            for column in df.columns:
                values = df[column].to_list()
                if type=='Usem':
                    #执行选多数的操作
                    #values_set = majority_classes(values)
                    if values[-1] == 0:
                        continue
                    values_set = [values[-1]]
                elif type=='Uset':
                    #去重操作
                    values_set = set(values)
                
                #如果这一列找不到Yago里的entity，那就跳过
                if len(values_set) == 0:
                    continue

                minHash = MinHash(num_perm=n)
                values_set = list(values_set)
                values_set = [str(value) for value in values_set]
                minHash.update_batch([value.encode('utf-8') for value in values_set])
                lsh.insert(filename+" "+column, minHash)
        if id==0:
            sys.stdout.write("\rId 0 Process Read and minhash {}/{} files".format(i+1, len(file_list)))
    if id==0:
        sys.stdout.write("\n")
    q.put(lsh)

def minhash_Lsh(type='Uset', folder_path = 'benchmark', t=0.9, n =128):
    lsh = MinHashLSH(threshold=t, num_perm=n)