| 71 | # 将csv文件列表中每个表格中的column转换为三元组(minhashes, sets, keys)形式 |
| 72 | # @profile |
| 73 | def bootstrap_sets(set_path, sets_files, num_perm): |
| 74 | # 从csv中读取加载 |
| 75 | print("Creating sets...") |
| 76 | print("Using num_perm = {}".format(num_perm)) |
| 77 | sizes = [] |
| 78 | keys = [] |
| 79 | # random.seed(rand_seed) |
| 80 | minhashes = dict() |
| 81 | ms = [] |
| 82 | for sets_file in sets_files: |
| 83 | df = pd.read_csv(os.path.join(set_path, sets_file), dtype='str').dropna() |
| 84 | columns = df.columns.tolist() |
| 85 | data = df.values.T.tolist() |
| 86 | for column,vals in zip(columns,data): |
| 87 | # if random.random() > sample_ratio: |
| 88 | # continue |
| 89 | # 需要对value去重 |
| 90 | vals = list(set(vals)) |
| 91 | # s = np.array(vals) |
| 92 | # sets.append(s) |
| 93 | # 域的键值 |
| 94 | keys.append(sets_file+"."+column) |
| 95 | sizes.append(len(vals)) |
| 96 | # 生成minhash |
| 97 | m = MinHash(num_perm, hashfunc=_hash_32) |
| 98 | for word in vals: |
| 99 | m.update(str(word)) |
| 100 | ms.append(m) |
| 101 | |
| 102 | sys.stdout.write("\rRead and minhashed {} sets".format(len(keys))) |
| 103 | sys.stdout.write("\n") |
| 104 | |
| 105 | # [hash函数个数][各集合] |
| 106 | minhashes[num_perm] = ms |
| 107 | |
| 108 | return (minhashes, sizes, keys) |
| 109 | |
| 110 | # 将csv文件列表中每个表格中的column转换为三元组(minhashes, sets, keys)形式 |
| 111 | # @profile |