Function deduplicate_entities

graphify/dedup.py:192–448

Deduplicate near-identical entities in a knowledge graph. Args: nodes: list of node dicts with at minimum {"id": str, "label": str} edges: list of edge dicts with {"source": str, "target": str, ...} communities: mapping of node_id -> community_id (from cluster())

(
    nodes: list[dict],
    edges: list[dict],
    *,
    communities: dict[str, int],
    dedup_llm_backend: str | None = None,
)

# ── main entry point ──────────────────────────────────────────────────────────

def deduplicate_entities(
    nodes: list[dict],
    edges: list[dict],
    *,
    communities: dict[str, int],
    dedup_llm_backend: str | None = None,
) -> tuple[list[dict], list[dict]]:
    """Deduplicate near-identical entities in a knowledge graph.

    Args:
        nodes: list of node dicts with at minimum {"id": str, "label": str}
        edges: list of edge dicts with {"source": str, "target": str, ...}
        communities: mapping of node_id -> community_id (from cluster())
        dedup_llm_backend: if set, use LLM to resolve ambiguous pairs

    Returns:
        (deduped_nodes, deduped_edges) with edges rewired to survivors
    """
    # Guard: cross-project dedup is not supported — nodes from different repos
    # share label names by coincidence and must never be merged by string similarity.
    # If you need to dedup a global graph, run deduplicate_entities per-repo first.
    repos_seen = {n.get("repo") for n in nodes if n.get("repo")}
    if len(repos_seen) > 1:
        raise ValueError(
            f"deduplicate_entities: nodes span multiple repos {sorted(repos_seen)!r}. "
            f"Cross-project dedup is disabled — run dedup per-repo before merging."
        )

    if len(nodes) <= 1:
        return nodes, edges

    # Pre-deduplicate: keep first occurrence of each id.
    # Warn when two nodes share an ID but originate from different source files —
    # this indicates a cross-chunk ID collision (#1504) where silent data loss occurs.
    seen_ids: dict[str, dict] = {}
    for node in nodes:
        nid = node.get("id", "")
        if not nid:
            continue
        if nid not in seen_ids:
            seen_ids[nid] = node
        else:
            existing_sf = seen_ids[nid].get("source_file") or ""
            new_sf = node.get("source_file") or ""
            if existing_sf != new_sf:
                print(
                    f"[graphify] WARNING: node '{nid}' from '{new_sf}' collides with "
                    f"node from '{existing_sf}' — the second node will be dropped. "
                    f"This is a cross-chunk ID collision caused by two files with the "
                    f"same name in different directories. To avoid data loss, run "
                    f"'graphify extract' per subfolder and merge with "
                    f"'graphify merge-graphs'.",
                    file=sys.stderr,
                )
    unique_nodes = list(seen_ids.values())

    if len(unique_nodes) <= 1:
        return unique_nodes, edges