Cluster the potential core components into modules. Args: completer: optional ``(prompt: str) -> str`` callable. When provided, clustering calls go through this completer instead of the legacy ``call_llm``. This is how the LLMBackend abstraction injects
(
leaf_nodes: List[str],
components: Dict[str, Node],
config: Config,
current_module_tree: dict[str, Any] = {},
current_module_name: str = None,
current_module_path: List[str] = [],
completer: Optional[Completer] = None,
)
| 54 | |
| 55 | |
| 56 | def cluster_modules( |
| 57 | leaf_nodes: List[str], |
| 58 | components: Dict[str, Node], |
| 59 | config: Config, |
| 60 | current_module_tree: dict[str, Any] = {}, |
| 61 | current_module_name: str = None, |
| 62 | current_module_path: List[str] = [], |
| 63 | completer: Optional[Completer] = None, |
| 64 | ) -> Dict[str, Any]: |
| 65 | """ |
| 66 | Cluster the potential core components into modules. |
| 67 | |
| 68 | Args: |
| 69 | completer: optional ``(prompt: str) -> str`` callable. When provided, |
| 70 | clustering calls go through this completer instead of the legacy |
| 71 | ``call_llm``. This is how the LLMBackend abstraction injects |
| 72 | subscription-mode (caw) routing. If ``None``, falls back to |
| 73 | ``call_llm`` for backward compatibility with direct callers. |
| 74 | """ |
| 75 | potential_core_components, potential_core_components_with_code = ( |
| 76 | format_potential_core_components(leaf_nodes, components) |
| 77 | ) |
| 78 | input_tokens = count_tokens(potential_core_components_with_code) |
| 79 | threshold = config.max_token_per_module |
| 80 | module_label = current_module_name or "repository" |
| 81 | |
| 82 | logger.info( |
| 83 | "Module clustering input for %s: %d leaf nodes, %d tokens, threshold %d", |
| 84 | module_label, |
| 85 | len(leaf_nodes), |
| 86 | input_tokens, |
| 87 | threshold, |
| 88 | ) |
| 89 | |
| 90 | if input_tokens <= threshold: |
| 91 | logger.info( |
| 92 | "Skipping LLM module clustering for %s because %d tokens fit within the " |
| 93 | "%d-token threshold; using whole-module documentation mode.", |
| 94 | module_label, |
| 95 | input_tokens, |
| 96 | threshold, |
| 97 | ) |
| 98 | return {} |
| 99 | |
| 100 | prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) |
| 101 | logger.info( |
| 102 | "Requesting LLM module clustering for %s because %d tokens exceed the %d-token threshold.", |
| 103 | module_label, |
| 104 | input_tokens, |
| 105 | threshold, |
| 106 | ) |
| 107 | if completer is not None: |
| 108 | response = completer(prompt) |
| 109 | else: |
| 110 | response = call_llm(prompt, config, model=config.cluster_model) |
| 111 | |
| 112 | #parse the response |
| 113 | try: |
no test coverage detected