()
| 156 | |
| 157 | #首先读取所有的表，循环处理，每次处理一个表 |
| 158 | def main(): |
| 159 | start_time = time.time() |
| 160 | with open("lsh/UsetLSH.pkl", "rb") as f: |
| 161 | u_set_lsh = pickle.load(f) |
| 162 | with open("lsh/UsemLSH.pkl", "rb") as f: |
| 163 | u_sem_lsh = pickle.load(f) |
| 164 | with open("lsh/UnlLSH.pkl", "rb") as f: |
| 165 | u_nl_lsh = pickle.load(f) |
| 166 | |
| 167 | df = pd.read_csv("groundtruth/att_groundtruth.csv") |
| 168 | column_data = df["query_table"] |
| 169 | unique_values = column_data.unique() |
| 170 | np.random.seed(41) |
| 171 | random_values = np.random.choice(unique_values, size=100, replace=False) |
| 172 | |
| 173 | query_tables = random_values.tolist() |
| 174 | end_time = time.time() |
| 175 | att_columns = ['query_table', 'candidate_table', 'query_col_name', 'candidate_col_name'] |
| 176 | alignment_columns = ['query_table', 'candidate_table', 'c'] |
| 177 | df_att = pd.DataFrame(columns=att_columns) |
| 178 | df_align = pd.DataFrame(columns=alignment_columns) |
| 179 | df_att.to_csv('tusResult/att_result.csv') |
| 180 | df_align.to_csv('tusResult/alignment_result.csv') |
| 181 | print('pre computed time: ' + str(end_time - start_time)+'s.') |
| 182 | #对每个query table求候选表 |
| 183 | for table in query_tables: |
| 184 | #分别加载 |
| 185 | #对每一列求候选的表格，所有候选表去重 |
| 186 | start_time = time.time() |
| 187 | Aunion = set() |
| 188 | #把这一段写成函数的形式 |
| 189 | get_candidate(table, u_set_lsh, 'benchmark', 5, Aunion, model, type='Uset', threshold=threshold) |
| 190 | time1 = time.time() |
| 191 | print('get uset candiate time: '+str(time1 - start_time)+'s.') |
| 192 | |
| 193 | get_candidate(table, u_sem_lsh, 'UsemLshTest', 5, Aunion, model, type='Usem', threshold=threshold) |
| 194 | time2 = time.time() |
| 195 | print('get usem candiate time: '+str(time2 - time1)+'s.') |
| 196 | |
| 197 | get_candidate(table, u_nl_lsh, 'benchmark', 5, Aunion, model, type = 'Unl', threshold=threshold) |
| 198 | time3 = time.time() |
| 199 | print('get unl candiate time: '+str(time3 - time2)+'s.') |
| 200 | |
| 201 | |
| 202 | #求出了Aunion,之后根据Aunion计算准确地Uensem，然后根据这个淘汰一部分，最后计算topk个表 |
| 203 | candiate_table_set = [] |
| 204 | for candiate_table in Aunion: |
| 205 | matching, score = alignment(table, candiate_table, 'cal') |
| 206 | candiate_table_set.append((matching, score)) |
| 207 | |
| 208 | end_time = time.time() |
| 209 | total_align_time = end_time-time3 |
| 210 | sorted_candiate_table_set = sorted(candiate_table_set, key = lambda x:x[1], reversed=True) |
| 211 | |
| 212 | print(table+' use time '+str(total_align_time)+' s.'+' average process a candiate time is '+str(total_align_time/len(Aunion))) |
| 213 | att_save = {} |
| 214 | for column in att_columns: |
| 215 | att_save[column] = [] |
no test coverage detected