| 349 | res.append(ret[0]) |
| 350 | return res |
| 351 | def get_text_units(working_dir,chunks_set,chunks_file,k=5): |
| 352 | db_name=os.path.basename(working_dir) |
| 353 | chunks_list=[] |
| 354 | for chunks in chunks_set: |
| 355 | if "|" in chunks: |
| 356 | temp_chunks=chunks.split("|") |
| 357 | else: |
| 358 | temp_chunks=[chunks] |
| 359 | chunks_list+=temp_chunks |
| 360 | counter = Counter(chunks_list) |
| 361 | |
| 362 | # 筛选出出现多次的元素 |
| 363 | # duplicates = [item for item, count in counter.items() if count > 2] |
| 364 | duplicates = [item for item, _ in sorted( |
| 365 | [(item, count) for item, count in counter.items() if count > 1], |
| 366 | key=lambda x: x[1], |
| 367 | reverse=True |
| 368 | )[:k]] |
| 369 | if len(duplicates)< k: |
| 370 | used = set(duplicates) |
| 371 | for item, _ in counter.items(): |
| 372 | if item not in used: |
| 373 | duplicates.append(item) |
| 374 | used.add(item) |
| 375 | if len(duplicates) == k: |
| 376 | break |
| 377 | |
| 378 | chunks_dict={} |
| 379 | text_units="" |
| 380 | with open (chunks_file,'r')as f: |
| 381 | chunks_dict= json.load(f) |
| 382 | chunks_dict={item["hash_code"]: item["text"] for item in chunks_dict} |
| 383 | |
| 384 | for chunks in duplicates: |
| 385 | text_units+=chunks_dict[chunks]+"\n" |
| 386 | return text_units |
| 387 | |
| 388 | def search_community(entity_name,working_dir): |
| 389 | db = pymysql.connect(host='localhost',port=4321, user='root', |