Function process_subset

union/TUS/Usem.py:164–203

(file_list)

    cal_list = [x for x in floder_list if x not in exist_list] 

    def process_subset(file_list):
        conn, cursor = get_yagoTypes_from_tsv()
        for file_name in file_list:
            if file_name.endswith('.csv') and file_name not in exist_list:
                file_path = os.path.join(floder_path, file_name)
                df = pd.read_csv(file_path)
                saved_data = {}

                for column in df.columns:
                    values = df[column].to_list()
                    values = [process_string(s) for s in values if s != ' ' and s != '']
                    counter = Counter(values)
                    values_set = set(values)

                    classes_of_thisColumn = []
                    for value in values_set:
                        search_keywords = value
                        results = query_lsh(lsh, search_keywords, 1)

                        for result in results:
                            query = f'''SELECT distinct taxonomy_class
                            FROM yagoTypejoinclasses
                            WHERE entity = '{result}';
                            '''
                            cursor.execute(query)
                            temp = cursor.fetchall()
                            for i in range(len(temp)):
                                temp[i] = temp[i][0]

                            classes_of_thisColumn += temp * counter[value]
                    saved_data[column] = classes_of_thisColumn

                saved_data = fill_zero(saved_data)
                saved_df = pd.DataFrame(saved_data)
                saved_file_path = os.path.join(save_floder_path, file_name)
                saved_df.to_csv(saved_file_path, index=False)
                conn.commit()
                cursor.close()
                conn.close()
                print('success saved on' + saved_file_path)
    cal_sublists = np.array_split(cal_list, 16)
    cal_sublists = [c.tolist() for c in cal_sublists]