| 36 | |
| 37 | |
| 38 | def querytable_columns(tablefile,column,inversed,KIV,KIA,docNo,pprdict): |
| 39 | df = pd.read_csv(tablefile,encoding= 'ISO-8859-1',low_memory=False,lineterminator='\n') |
| 40 | columnvalues = df[column].unique() |
| 41 | |
| 42 | columnvalues_len = len(columnvalues) |
| 43 | |
| 44 | KIA_docs_set = set(KIA[column]) |
| 45 | |
| 46 | KIV_docs_list = [] |
| 47 | for value in columnvalues: |
| 48 | KIV_docs_list += inversed[value] |
| 49 | |
| 50 | KIV_docs_set = set(KIV_docs_list) |
| 51 | |
| 52 | KIV_KIA_set = KIA_docs_set & KIV_docs_set |
| 53 | |
| 54 | # compute weight |
| 55 | weigth_dict = {} |
| 56 | for doc in KIV_KIA_set: |
| 57 | docvalue_set = set(KIV[doc]) |
| 58 | doclen = len(docvalue_set) |
| 59 | overlapevalues = docvalue_set & set(columnvalues) |
| 60 | weight = len(overlapevalues) / min(doclen,columnvalues_len) |
| 61 | weigth_dict[doc] = weight |
| 62 | |
| 63 | |
| 64 | # get all the weigted dict |
| 65 | total_ppr_list = [] |
| 66 | for doc in KIV_KIA_set: |
| 67 | doc_weigth = weigth_dict[doc] |
| 68 | doc_dict_weigth = {key:value*doc_weigth for key,value in pprdict[doc].items()} |
| 69 | total_ppr_list.append(doc_dict_weigth) |
| 70 | |
| 71 | # compute the weighted dict |
| 72 | total_ppr = defaultdict(float) |
| 73 | for ppr_ele in total_ppr_list: |
| 74 | for key,value in ppr_ele.items(): |
| 75 | total_ppr[key] += value |
| 76 | |
| 77 | sorted_dict = dict(sorted(total_ppr.items(), key=lambda x: x[1],reverse=True)) |
| 78 | |
| 79 | return_doc_list = list(sorted_dict.keys())[:30] |
| 80 | |
| 81 | return_name = [] |
| 82 | for docnumber in return_doc_list: |
| 83 | return_name.append(docNo[docnumber]) |
| 84 | |
| 85 | return return_name |
| 86 | # queryfilepath = "/data_ssd/webtable/large/split_1/csvData10000001.csv" |
| 87 | # columnname= "Sample Date" |
| 88 | # res = querytable_columns(queryfilepath,columnname,inversed,KIV,KIA,graph,docNo) |