| 9 | |
| 10 | |
| 11 | class MachineLearnSample: |
| 12 | def __init__(self,index): |
| 13 | self.KIV = index.KIV |
| 14 | self.inversed_index = index.inversed_index |
| 15 | self.doc_number = index.doc_number |
| 16 | |
| 17 | def computer_pair_docs_label_company(self,company): |
| 18 | doc1, doc2 = company |
| 19 | term_doc1 = self.KIV[doc1] |
| 20 | term_doc2 = self.KIV[doc2] |
| 21 | doc1_doc2_term_union = term_doc1 | term_doc2 |
| 22 | keyterm = "" |
| 23 | min_docset_len = 2000000000 |
| 24 | for term in doc1_doc2_term_union: |
| 25 | set_len = len(self.inversed_index[term]) |
| 26 | if set_len < min_docset_len: |
| 27 | keyterm = term |
| 28 | min_docset_len = set_len |
| 29 | if 0 < min_docset_len < 20: |
| 30 | break |
| 31 | |
| 32 | min_docset = self.inversed_index[keyterm] - {doc1, doc2} |
| 33 | for docnum in min_docset: |
| 34 | if (doc1_doc2_term_union <= self.KIV[docnum]): |
| 35 | return (doc1, doc2, docnum) |
| 36 | return (doc1, doc2, -1) |
| 37 | |
| 38 | def getlabled(self): |
| 39 | startime = time.time() |
| 40 | matix_len = np.array([i for i in range(self.doc_number)]) |
| 41 | |
| 42 | with Pool(2) as p: |
| 43 | data_infos = map(self.computer_pair_docs_label_company, itertools.combinations(matix_len, 2)) |
| 44 | |
| 45 | result = [tup for tup in list(data_infos) if tup[2] != -1] |
| 46 | endtime = time.time() |
| 47 | time_consum = endtime - startime |
| 48 | print("计算full labled的时间为：%s" % time_consum) |
| 49 | print("changdu:%s" % len(result)) |
| 50 | |
| 51 | with open("D:\ljj\data\similiarmatrix.pickle", 'wb') as f: |
| 52 | pickle.dump(result, f) |
| 53 | print("full labled 已经存储") |
| 54 | |
| 55 | |
| 56 |
nothing calls this directly
no outgoing calls
no test coverage detected