(
self,
global_config: dict,
entities: dict,
relations:dict,
max_length_in_cluster: int = 60000,
tokenizer=tiktoken.get_encoding("cl100k_base"),
reduction_dimension: int = 2,
cluster_threshold: float = 0.1,
verbose: bool = False,
threshold: float = 0.98, # 0.99
thredshold_change_rate: float = 0.05,
WORKING_DIR: str = None,
max_workers: int =8,
cluster_size: int=20,
)
| 499 | |
| 500 | class Hierarchical_Clustering(ClusteringAlgorithm): |
| 501 | def perform_clustering( |
| 502 | self, |
| 503 | global_config: dict, |
| 504 | entities: dict, |
| 505 | relations:dict, |
| 506 | max_length_in_cluster: int = 60000, |
| 507 | tokenizer=tiktoken.get_encoding("cl100k_base"), |
| 508 | reduction_dimension: int = 2, |
| 509 | cluster_threshold: float = 0.1, |
| 510 | verbose: bool = False, |
| 511 | threshold: float = 0.98, # 0.99 |
| 512 | thredshold_change_rate: float = 0.05, |
| 513 | WORKING_DIR: str = None, |
| 514 | max_workers: int =8, |
| 515 | cluster_size: int=20, |
| 516 | ) -> List[dict]: |
| 517 | use_llm_func: callable = global_config["use_llm_func"] |
| 518 | embeddings_func: callable = global_config["embeddings_func"] |
| 519 | # Get the embeddings from the nodes |
| 520 | nodes = list(entities.values()) |
| 521 | embeddings = np.array([x["vector"] for x in nodes]) |
| 522 | generate_relations={} |
| 523 | max_workers=global_config['max_workers'] |
| 524 | community_report={} |
| 525 | all_nodes=[] |
| 526 | all_nodes.append(nodes) |
| 527 | community_report_prompt = PROMPTS["aggregate_entities"] |
| 528 | cluster_cluster_relation_prompt = PROMPTS["cluster_cluster_relation"] |
| 529 | max_depth=round(math.log(len(nodes),cluster_size))+1 |
| 530 | for layer in range(max_depth): |
| 531 | logging.info(f"############ Layer[{layer}] Clustering ############") |
| 532 | # Perform the clustering |
| 533 | if len(nodes) <= 2: |
| 534 | print("当前簇数小于2，停止聚类") |
| 535 | break |
| 536 | clusters = perform_clustering( |
| 537 | embeddings, dim=reduction_dimension, threshold=cluster_threshold,cluster_size=cluster_size |
| 538 | ) |
| 539 | temp_clusters_nodes = [] |
| 540 | # Initialize an empty list to store the clusters of nodes |
| 541 | # Iterate over each unique label in the clusters |
| 542 | unique_clusters = np.unique(np.concatenate(clusters)) |
| 543 | logging.info(f"[Clustered Label Num: {len(unique_clusters)} / Last Layer Total Entity Num: {len(nodes)}]") |
| 544 | # calculate the number of nodes belong to each cluster |
| 545 | # cluster_sizes = Counter(np.concatenate(clusters)) |
| 546 | # # calculate cluster sparsity |
| 547 | # cluster_sparsity = 1 - sum([x * (x - 1) for x in cluster_sizes.values()])/(len(nodes) * (len(nodes) - 1)) |
| 548 | # cluster_sparsity_change_rate = (abs(cluster_sparsity - pre_cluster_sparsity) / pre_cluster_sparsity) |
| 549 | # pre_cluster_sparsity = cluster_sparsity |
| 550 | # logging.info(f"[Cluster Sparsity: {round(cluster_sparsity, 4) * 100}%]") |
| 551 | |
| 552 | # if cluster_sparsity_change_rate <= thredshold_change_rate: |
| 553 | # logging.info(f"[Stop Clustering at Layer{layer} with Cluster Sparsity Change Rate {round(cluster_sparsity_change_rate, 4) * 100}%]") |
| 554 | # break |
| 555 | # summarize |
| 556 | if len(unique_clusters) <=4: |
| 557 | print(f"当前簇数小于5，停止聚类") |
| 558 | break |
no test coverage detected