(type='Uset', folder_path = 'benchmark', t=0.9, n =128)
| 155 | q.put(lsh) |
| 156 | |
| 157 | def minhash_Lsh(type='Uset', folder_path = 'benchmark', t=0.9, n =128): |
| 158 | lsh = MinHashLSH(threshold=t, num_perm=n) |
| 159 | split_num = 40 |
| 160 | #with open('/data_ssd/tus/webtable/lsh/UsetLSH_split_2.pkl', "rb") as f: |
| 161 | #lsh = pickle.load(f) |
| 162 | #child_folders = ['split_'+str(i) for i in range(1,7)] |
| 163 | #child_folders = ['datasets_CAN', 'datasets_SG', 'datasets_UK', 'datasets_USA'] |
| 164 | child_folders = ['datasets_CAN'] |
| 165 | file_list = [] |
| 166 | for child_folder in child_folders: |
| 167 | files = os.listdir(folder_path+child_folder) |
| 168 | files = [os.path.join(folder_path+child_folder, f) for f in files] |
| 169 | file_list.extend(files) |
| 170 | #file_list = os.listdir(folder_path) |
| 171 | spl = folder_path.split('/')[-1] |
| 172 | |
| 173 | # 读入候选集目录 |
| 174 | sub_sets_files = split_list(file_list, split_num) |
| 175 | process_list = [] |
| 176 | # pocessing |
| 177 | entries_q = multiprocessing.Queue() |
| 178 | for i in range(split_num): |
| 179 | process = multiprocessing.Process(target=minhash_multi_process, args=(folder_path, sub_sets_files[i], entries_q, spl, i, n, type, t)) |
| 180 | process.daemon = True |
| 181 | process_list.append(process) |
| 182 | process.start() |
| 183 | for i in range(split_num): |
| 184 | lsh.union(entries_q.get()) |
| 185 | sys.stdout.write("\rRead and minhash {}/{} sets".format(i+1,split_num)) |
| 186 | sys.stdout.write("\n") |
| 187 | for i, process in enumerate(process_list): |
| 188 | process.join() |
| 189 | |
| 190 | # for filename in tqdm(file_list): |
| 191 | # if filename.endswith('.csv'): |
| 192 | # file_path = os.path.join(folder_path, filename) |
| 193 | # df = pd.read_csv(file_path) |
| 194 | # #取出每一列，将每一列的值minhah，然后加入lsh中，创建索引，列的索引用文件名+列名 |
| 195 | # for column in df.columns: |
| 196 | # values = df[column].to_list() |
| 197 | # if type=='Usem': |
| 198 | # #执行选多数的操作 |
| 199 | # #values_set = majority_classes(values) |
| 200 | # if values[-1] == 0: |
| 201 | # continue |
| 202 | # values_set = [values[-1]] |
| 203 | # elif type=='Uset': |
| 204 | # #去重操作 |
| 205 | # values_set = set(values) |
| 206 | |
| 207 | # #如果这一列找不到Yago里的entity，那就跳过 |
| 208 | # if len(values_set) == 0: |
| 209 | # continue |
| 210 | |
| 211 | # minHash = MinHash(num_perm=n) |
| 212 | # values_set = list(values_set) |
| 213 | # values_set = [str(value) for value in values_set] |
| 214 | # minHash.update_batch([value.encode('utf-8') for value in values_set]) |
no test coverage detected